Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

48.1 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""is_vision_model() caches True, False, subprocess and exception-fallback results per model."""

import struct
import sys
import types as _types
from pathlib import Path
from unittest.mock import patch, MagicMock

import pytest


def _shared_setup_1(monkeypatch):
    import utils.models.model_config as mc

    mc._audio_detection_cache.clear()
    monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
    monkeypatch.setattr(mc, "resolve_cached_repo_id_case", lambda n, *_a, **_k: n)
    return mc


_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
    sys.path.insert(0, _BACKEND_DIR)

_loggers_stub = _types.ModuleType("loggers")
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
sys.modules.setdefault("loggers", _loggers_stub)

from utils.models.model_config import (
    ModelConfig,
    load_model_config,
    is_vision_model,
    _detect_audio_from_tokenizer,
    _is_vision_model_subprocess,
    _is_vision_model_uncached,
    _vision_detection_cache,
)


@pytest.fixture(autouse = True)
def _clear_vision_cache(tmp_path, monkeypatch):
    """Run each test from an empty cwd: a relative model id that exists on disk counts as local."""
    monkeypatch.chdir(tmp_path)
    _vision_detection_cache.clear()
    yield
    _vision_detection_cache.clear()


class TestVisionCacheHitMiss:
    """Verify the cache prevents redundant detection calls."""

    @patch("utils.models.model_config._is_vision_model_uncached", return_value = True)
    def test_second_call_uses_cache(self, mock_uncached):
        """Two calls for the same model invoke the uncached fn once."""
        assert is_vision_model("org/my-vlm") is True
        assert is_vision_model("org/my-vlm") is True
        mock_uncached.assert_called_once_with("org/my-vlm", None, local_files_only = False)

    @patch("utils.models.model_config._is_vision_model_uncached", return_value = False)
    def test_different_models_each_detected(self, mock_uncached):
        """Different model names should each trigger detection."""
        is_vision_model("model-a")
        is_vision_model("model-b")
        assert mock_uncached.call_count == 2

    @patch("utils.models.model_config._is_vision_model_uncached", return_value = True)
    def test_cache_returns_correct_value(self, mock_uncached):
        """The cached value must match what _is_vision_model_uncached returned."""
        first = is_vision_model("org/vlm")
        second = is_vision_model("org/vlm")
        assert first is True
        assert second is True


class TestVisionCacheStoresFalse:
    """Non-VLM results (False) must also be cached to avoid re-detection."""

    @patch("utils.models.model_config._is_vision_model_uncached", return_value = False)
    def test_false_result_cached(self, mock_uncached):
        assert is_vision_model("org/text-only") is False
        assert is_vision_model("org/text-only") is False
        mock_uncached.assert_called_once()
        assert _vision_detection_cache[("org/text-only", None, False)] is False


class TestVisionCacheSubprocessPath:
    """transformers 5.x models go through _is_vision_model_subprocess, which the cache spawns at
    most once per model per process."""

    @patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
    @patch("utils.models.model_config._is_vision_model_subprocess", return_value = True)
    @patch("utils.transformers_version.needs_transformers_5", return_value = True)
    def test_subprocess_called_once_with_cache(self, mock_needs_t5, mock_subprocess, mock_raw):
        """An inconclusive raw config runs the 5.x subprocess once; the second call is cached."""
        assert is_vision_model("unsloth/Qwen3.5-2B") is True
        assert is_vision_model("unsloth/Qwen3.5-2B") is True

        mock_subprocess.assert_called_once()
        assert _vision_detection_cache[("unsloth/Qwen3.5-2B", None, False)] is True

    @patch("utils.models.model_config._raw_config_has_vision_config", return_value = True)
    @patch("utils.models.model_config._is_vision_model_subprocess", return_value = None)
    @patch("utils.transformers_version.needs_transformers_5", return_value = True)
    def test_raw_config_primary_skips_subprocess(
        self, mock_needs_t5, mock_subprocess, mock_raw_config
    ):
        # The raw config.json read is the primary path; a definitive answer there never reaches the subprocess.
        assert is_vision_model("unsloth/gemma-4-E4B-it") is True
        assert is_vision_model("unsloth/gemma-4-E4B-it") is True

        mock_raw_config.assert_called_once_with(
            "unsloth/gemma-4-E4B-it", hf_token = None, local_files_only = False
        )
        mock_subprocess.assert_not_called()


def _projector_declaring(path: Path, key: str) -> Path:
    """A minimal GGUF carrying one ``clip.has_*_encoder`` bool, no tensors."""
    kv = struct.pack("<Q", len(key)) + key.encode() + struct.pack("<I", 7) + struct.pack("<?", True)
    path.write_bytes(struct.pack("<IIQQ", 0x46554747, 3, 0, 1) + kv)
    return path


class TestLocalGgufVisionDetection:
    """Every projector fixture is non-empty, since ``detect_mmproj_file`` skips a zero-byte one
    as an interrupted download; those built by ``_projector_declaring`` also carry a header,
    because the capability they assert is read from it."""

    @patch(
        "utils.models.model_config._is_vision_model_subprocess",
        side_effect = AssertionError("GGUF must not use Transformers vision detection"),
    )
    def test_qwen36_gguf_with_mmproj_skips_transformers(self, mock_subprocess, tmp_path):
        model = tmp_path / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
        model.write_bytes(b"")
        (tmp_path / "mmproj-F32.gguf").write_bytes(b"\0" * 32)

        assert is_vision_model(str(model)) is True
        mock_subprocess.assert_not_called()

    @patch(
        "utils.models.model_config._is_vision_model_subprocess",
        side_effect = AssertionError("GGUF must not use Transformers vision detection"),
    )
    def test_direct_gguf_in_variant_subdir_finds_snapshot_mmproj(self, mock_subprocess, tmp_path):
        variant_dir = tmp_path / "BF16"
        variant_dir.mkdir()
        model = variant_dir / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
        model.write_bytes(b"")
        (tmp_path / "mmproj-F32.gguf").write_bytes(b"\0" * 32)

        assert is_vision_model(str(model)) is True
        mock_subprocess.assert_not_called()

    @patch(
        "utils.models.model_config._is_vision_model_subprocess",
        side_effect = AssertionError("GGUF must not use Transformers vision detection"),
    )
    def test_qwen36_gguf_without_mmproj_skips_transformers(self, mock_subprocess, tmp_path):
        model = tmp_path / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
        model.write_bytes(b"")

        assert is_vision_model(str(model)) is False
        mock_subprocess.assert_not_called()

    def test_local_gguf_check_observes_mmproj_added_later(self, tmp_path):
        model = tmp_path / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
        model.write_bytes(b"")

        assert is_vision_model(str(model)) is False
        (tmp_path / "mmproj-F32.gguf").write_bytes(b"\0" * 32)
        assert is_vision_model(str(model)) is True

    @patch(
        "utils.models.model_config._is_vision_model_subprocess",
        side_effect = AssertionError("GGUF must not use Transformers vision detection"),
    )
    def test_ui_selection_returns_local_gguf_config(self, mock_subprocess, tmp_path):
        model = tmp_path / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
        model.write_bytes(b"")
        mmproj = tmp_path / "mmproj-F32.gguf"
        mmproj.write_bytes(b"\0" * 32)

        config = ModelConfig.from_ui_selection(str(model), None)

        assert config is not None
        assert config.is_gguf is True
        assert config.is_vision is True
        assert config.gguf_mmproj_file == str(mmproj.resolve())
        mock_subprocess.assert_not_called()

    @patch(
        "utils.models.model_config._is_vision_model_subprocess",
        side_effect = AssertionError("GGUF must not use Transformers vision detection"),
    )
    def test_ui_selection_direct_gguf_in_variant_subdir_keeps_mmproj(
        self, mock_subprocess, tmp_path
    ):
        variant_dir = tmp_path / "BF16"
        variant_dir.mkdir()
        model = variant_dir / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
        model.write_bytes(b"")
        mmproj = tmp_path / "mmproj-F32.gguf"
        mmproj.write_bytes(b"\0" * 32)

        config = ModelConfig.from_ui_selection(str(model), None)

        assert config is not None
        assert config.is_gguf is True
        assert config.is_vision is True
        assert config.gguf_mmproj_file == str(mmproj.resolve())
        mock_subprocess.assert_not_called()

    def test_an_audio_only_projector_is_not_a_vision_model(self, tmp_path):
        """ultravox, Voxtral and Qwen3-ASR take audio projectors only, so no image input."""
        model = tmp_path / "Voxtral-Mini-3B-2507-Q4_K_M.gguf"
        model.write_bytes(b"\0" * 32)
        _projector_declaring(tmp_path / "mmproj-F16.gguf", "clip.has_audio_encoder")

        assert is_vision_model(str(model)) is False

    def test_a_projector_declaring_vision_is_still_a_vision_model(self, tmp_path):
        model = tmp_path / "Qwen3-VL-8B-Instruct-Q4_K_M.gguf"
        model.write_bytes(b"\0" * 32)
        _projector_declaring(tmp_path / "mmproj-F16.gguf", "clip.has_vision_encoder")

        assert is_vision_model(str(model)) is True

    @patch(
        "utils.models.model_config._is_vision_model_subprocess",
        side_effect = AssertionError("GGUF must not use Transformers vision detection"),
    )
    def test_named_quant_in_a_subdir_reads_the_snapshot_projector(self, mock_subprocess, tmp_path):
        """A repo whose quants all live under a per-quant subdir has no weight file at the
        snapshot root, which is the only place the root-level detector looks (#8772)."""
        variant_dir = tmp_path / "UD-Q4_K_XL"
        variant_dir.mkdir()
        (variant_dir / "Qwen3-VL-235B-UD-Q4_K_XL-00001-of-00002.gguf").write_bytes(b"\0" * 32)
        (variant_dir / "Qwen3-VL-235B-UD-Q4_K_XL-00002-of-00002.gguf").write_bytes(b"\0" * 32)
        (tmp_path / "mmproj-F32.gguf").write_bytes(b"\0" * 32)

        assert is_vision_model(str(tmp_path), gguf_variant = "UD-Q4_K_XL") is True
        mock_subprocess.assert_not_called()

    @patch(
        "utils.models.model_config._is_vision_model_subprocess",
        side_effect = AssertionError("GGUF must not use Transformers vision detection"),
    )
    def test_named_quant_in_a_subdir_without_a_projector_is_text_only(
        self, mock_subprocess, tmp_path
    ):
        variant_dir = tmp_path / "UD-Q4_K_XL"
        variant_dir.mkdir()
        (variant_dir / "Qwen3-235B-UD-Q4_K_XL.gguf").write_bytes(b"\0" * 32)

        assert is_vision_model(str(tmp_path), gguf_variant = "UD-Q4_K_XL") is False
        mock_subprocess.assert_not_called()

    @pytest.mark.parametrize(
        "variant, expected",
        [("Q4_K_M", True), ("Q8_0", False)],
    )
    def test_each_quant_answers_what_a_load_of_that_quant_would_see(
        self, tmp_path, variant, expected
    ):
        """One quant keeps the projector beside it and the other does not, so a probe that
        reads any quant of the directory answers one of them wrongly."""
        variant_dir = tmp_path / "Q4_K_M"
        variant_dir.mkdir()
        (variant_dir / "Qwen3-VL-8B-Instruct-Q4_K_M.gguf").write_bytes(b"\0" * 32)
        (variant_dir / "mmproj-F16.gguf").write_bytes(b"\0" * 32)
        (tmp_path / "Qwen3-VL-8B-Instruct-Q8_0.gguf").write_bytes(b"\0" * 32)

        config = ModelConfig.from_identifier(str(tmp_path), gguf_variant = variant)

        assert config is not None
        assert config.is_vision is expected
        assert is_vision_model(str(tmp_path), gguf_variant = variant) is expected

    @patch("utils.models.model_config._is_vision_model_uncached", return_value = False)
    def test_a_quant_that_is_not_on_disk_is_not_answered_by_another_one(
        self, mock_uncached, tmp_path
    ):
        """A load of an absent quant resolves no GGUF at all, so neither may the probe: the
        projector beside the quant that IS on disk says nothing about the one asked for."""
        (tmp_path / "Qwen3-VL-8B-Instruct-Q8_0.gguf").write_bytes(b"\0" * 32)
        (tmp_path / "mmproj-F16.gguf").write_bytes(b"\0" * 32)

        config = ModelConfig.from_identifier(str(tmp_path), gguf_variant = "UD-Q4_K_XL")

        assert config is not None
        assert config.is_gguf is False
        assert is_vision_model(str(tmp_path), gguf_variant = "UD-Q4_K_XL") is False


class TestVisionCacheOnException:
    """On exception, _is_vision_model_uncached distinguishes permanent
    failures (cached as False) from transient ones (returned as None,
    not cached, so the next call retries). Verify both contracts."""

    @patch(
        "utils.models.model_config.load_model_config",
        side_effect = ValueError("bad config"),
    )
    @patch("utils.transformers_version.needs_transformers_5", return_value = False)
    def test_permanent_exception_result_cached(self, mock_needs_t5, mock_load_config):
        """Permanent failures (e.g. ValueError, GatedRepoError) cache False so later calls do not retry."""
        assert is_vision_model("broken/model") is False
        assert is_vision_model("broken/model") is False
        mock_load_config.assert_called_once()

    @patch(
        "utils.models.model_config.load_model_config",
        side_effect = OSError("network down"),
    )
    @patch("utils.transformers_version.needs_transformers_5", return_value = False)
    def test_transient_exception_not_cached(self, mock_needs_t5, mock_load_config):
        """A transient failure (OSError, timeouts) returns None from
        _is_vision_model_uncached, surfaces as False, and is NOT cached
        so the next call retries."""
        assert is_vision_model("broken/model") is False
        assert is_vision_model("broken/model") is False
        assert mock_load_config.call_count == 2


class TestVisionCacheDirectPath:
    """Non-5.x models detect via load_model_config; the cache must handle them the same way."""

    @patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
    @patch("utils.transformers_version.needs_transformers_5", return_value = False)
    @patch("utils.models.model_config.load_model_config")
    def test_direct_vlm_detection_cached(self, mock_load_config, mock_needs_t5, mock_raw):
        """A standard VLM detected via architecture suffix should be cached."""
        cfg = MagicMock(spec = [])
        cfg.model_type = "gemma3"
        cfg.architectures = ["Gemma3ForConditionalGeneration"]
        mock_load_config.return_value = cfg

        assert is_vision_model("google/gemma-3-4b-it") is True
        assert is_vision_model("google/gemma-3-4b-it") is True
        mock_load_config.assert_called_once()

    @patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
    @patch("utils.transformers_version.needs_transformers_5", return_value = False)
    @patch("utils.models.model_config.load_model_config")
    def test_direct_non_vlm_detection_cached(self, mock_load_config, mock_needs_t5, mock_raw):
        """A standard text model (no VLM indicators) should cache False."""
        cfg = MagicMock(spec = [])
        cfg.model_type = "llama"
        cfg.architectures = ["LlamaForCausalLM"]
        mock_load_config.return_value = cfg

        assert is_vision_model("meta-llama/Llama-3-8B") is False
        assert is_vision_model("meta-llama/Llama-3-8B") is False
        mock_load_config.assert_called_once()

    @patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
    @patch("utils.transformers_version.needs_transformers_5", return_value = False)
    @patch("utils.models.model_config.load_model_config")
    def test_vision_config_attr_detected_and_cached(
        self, mock_load_config, mock_needs_t5, mock_raw
    ):
        """Models with vision_config (LLaVA, Qwen2-VL, etc.) should be cached as True."""
        cfg = MagicMock(spec = [])
        cfg.model_type = "qwen2_vl"
        cfg.architectures = ["Qwen2VLForCausalLM"]
        cfg.vision_config = {"hidden_size": 1024}
        mock_load_config.return_value = cfg

        assert is_vision_model("Qwen/Qwen2-VL-7B") is True
        assert is_vision_model("Qwen/Qwen2-VL-7B") is True
        mock_load_config.assert_called_once()

    @patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
    @patch("utils.transformers_version.needs_transformers_5", return_value = False)
    @patch("utils.models.model_config.load_model_config")
    def test_gemma4_model_type_detected_and_cached(self, mock_load_config, mock_needs_t5, mock_raw):
        cfg = MagicMock(spec = [])
        cfg.model_type = "gemma4"
        cfg.architectures = ["Gemma4ForConditionalGeneration"]
        mock_load_config.return_value = cfg

        assert is_vision_model("google/gemma-4-E4B-it") is True
        assert is_vision_model("google/gemma-4-E4B-it") is True
        mock_load_config.assert_called_once()

    @patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
    @patch("utils.transformers_version.needs_transformers_5", return_value = False)
    @patch("utils.models.model_config.load_model_config")
    def test_gemma4_audio_subconfig_not_detected_as_vision(
        self, mock_load_config, mock_needs_t5, mock_raw
    ):
        cfg = MagicMock(spec = [])
        cfg.model_type = "gemma4_audio"
        cfg.architectures = ["Gemma4AudioModel"]
        mock_load_config.return_value = cfg

        assert is_vision_model("local/gemma4-audio-encoder") is False
        assert is_vision_model("local/gemma4-audio-encoder") is False
        mock_load_config.assert_called_once()

    @patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
    @patch("utils.transformers_version.needs_transformers_5", return_value = False)
    @patch("utils.models.model_config.load_model_config")
    def test_gemma4_text_subconfig_not_detected_as_vision(
        self, mock_load_config, mock_needs_t5, mock_raw
    ):
        cfg = MagicMock(spec = [])
        cfg.model_type = "gemma4_text"
        cfg.architectures = ["Gemma4ForCausalLM"]
        mock_load_config.return_value = cfg

        assert is_vision_model("local/gemma-4-text") is False
        assert is_vision_model("local/gemma-4-text") is False
        mock_load_config.assert_called_once()

    @patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
    @patch("utils.transformers_version.needs_transformers_5", return_value = False)
    @patch("utils.models.model_config.load_model_config")
    def test_audio_model_excluded_and_cached(self, mock_load_config, mock_needs_t5, mock_raw):
        """Audio-only models (csm, whisper) with ForConditionalGeneration
        should be excluded from VLM detection and cached as False."""
        cfg = MagicMock(spec = [])
        cfg.model_type = "whisper"
        cfg.architectures = ["WhisperForConditionalGeneration"]
        mock_load_config.return_value = cfg

        assert is_vision_model("openai/whisper-large-v3") is False
        assert is_vision_model("openai/whisper-large-v3") is False
        mock_load_config.assert_called_once()


class TestVisionCacheTokenHandling:
    """Cache is keyed on (model_name, hf_token), so each token runs its own gated-model check."""

    @patch("utils.models.model_config._is_vision_model_uncached", return_value = True)
    def test_different_tokens_trigger_new_detection(self, mock_uncached):
        """An unauthenticated False for a gated model must not block the later authenticated check."""
        assert is_vision_model("gated/model", hf_token = "token-a") is True
        assert is_vision_model("gated/model", hf_token = "token-b") is True
        assert mock_uncached.call_count == 2

    @patch("utils.models.model_config._is_vision_model_uncached", return_value = True)
    def test_same_token_uses_cache(self, mock_uncached):
        """Repeated calls with identical model + token should hit cache."""
        assert is_vision_model("gated/model", hf_token = "token-a") is True
        assert is_vision_model("gated/model", hf_token = "token-a") is True
        mock_uncached.assert_called_once()


class TestRevisionAwareVisionDetection:
    """A pinned Hub commit must flow through every vision config read and cache key."""

    @patch(
        "utils.models.model_config._is_vision_model_uncached",
        side_effect = [False, True],
    )
    def test_different_revisions_do_not_share_cache(self, mock_uncached, monkeypatch):
        monkeypatch.setattr("utils.models.model_config._env_offline", lambda: False)
        assert is_vision_model("org/model", revision = "commit-a") is False
        assert is_vision_model("org/model", revision = "commit-b") is True
        assert is_vision_model("org/model", revision = "commit-a") is False

        assert mock_uncached.call_count == 2
        mock_uncached.assert_any_call(
            "org/model",
            None,
            local_files_only = False,
            revision = "commit-a",
        )
        mock_uncached.assert_any_call(
            "org/model",
            None,
            local_files_only = False,
            revision = "commit-b",
        )
        assert _vision_detection_cache[("org/model", None, False, "commit-a")] is False
        assert _vision_detection_cache[("org/model", None, False, "commit-b")] is True

    @patch("transformers.AutoConfig.from_pretrained")
    def test_load_model_config_forwards_only_non_null_revision(self, from_pretrained):
        load_model_config("org/model", use_auth = True, revision = "commit-a")
        assert from_pretrained.call_args.kwargs["revision"] == "commit-a"

        load_model_config("org/model", use_auth = True)
        assert "revision" not in from_pretrained.call_args.kwargs

    def test_raw_config_download_uses_revision(self, monkeypatch, tmp_path):
        import utils.models.model_config as mc

        config_path = tmp_path / "config.json"
        config_path.write_text('{"model_type": "llama"}')
        download = MagicMock(return_value = str(config_path))
        monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
        monkeypatch.setattr("huggingface_hub.hf_hub_download", download)

        assert mc._raw_config_has_vision_config("org/model", revision = "commit-a") is False
        assert download.call_args.kwargs["revision"] == "commit-a"

        assert mc._raw_config_has_vision_config("org/model") is False
        assert "revision" not in download.call_args.kwargs

    @patch("utils.models.model_config.load_model_config")
    @patch("utils.transformers_version.needs_transformers_5", return_value = False)
    @patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
    def test_direct_config_fallback_uses_revision(self, mock_raw, mock_needs_t5, mock_load):
        cfg = MagicMock(spec = [])
        cfg.model_type = "llama"
        cfg.architectures = ["LlamaForCausalLM"]
        mock_load.return_value = cfg

        assert (
            _is_vision_model_uncached(
                "org/model",
                hf_token = "hf_x",
                revision = "commit-a",
            )
            is False
        )
        mock_load.assert_called_once_with(
            "org/model",
            use_auth = True,
            token = "hf_x",
            local_files_only = False,
            revision = "commit-a",
        )

    @patch("utils.models.model_config._is_vision_model_subprocess", return_value = True)
    @patch("utils.transformers_version.needs_transformers_5", return_value = True)
    @patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
    def test_transformers_5_fallback_uses_revision(self, mock_raw, mock_needs_t5, mock_subprocess):
        assert (
            _is_vision_model_uncached(
                "org/model",
                hf_token = "hf_x",
                revision = "commit-a",
            )
            is True
        )
        mock_subprocess.assert_called_once_with(
            "org/model",
            hf_token = "hf_x",
            revision = "commit-a",
        )

    @patch("utils.transformers_version.get_transformers_tier", return_value = "default")
    @patch("utils.models.model_config.subprocess.run")
    def test_subprocess_command_carries_revision(self, run, mock_tier):
        run.return_value = MagicMock(
            returncode = 0,
            stdout = '{"is_vision": false}',
            stderr = "",
        )

        assert (
            _is_vision_model_subprocess(
                "org/model",
                hf_token = "hf_x",
                revision = "commit-a",
            )
            is False
        )
        # The token rides in the child env only; its argv slot stays, empty.
        assert run.call_args.args[0][-3:] == ["org/model", "", "commit-a"]
        assert run.call_args.kwargs["env"]["HF_TOKEN"] == "hf_x"
        assert 'kwargs["revision"] = revision' in run.call_args.args[0][2]


class TestVisionCacheLocalOnly:
    """local_files_only is in the cache key: an offline negative must not be reused by a
    later online probe (else a VLM is routed through the text loader until restart)."""

    def test_local_only_negative_does_not_poison_online(self, monkeypatch):
        import utils.models.model_config as mc

        mc._vision_detection_cache.clear()
        monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
        monkeypatch.setattr(mc, "resolve_cached_repo_id_case", lambda n, *_a, **_k: n)
        # Pin env-offline off so the key tracks the kwarg.
        monkeypatch.setattr(mc, "_env_offline", lambda: False)

        seen = []

        def _probe(
            name,
            hf_token = None,
            local_files_only = False,
        ):
            seen.append(local_files_only)
            return False if local_files_only else True

        monkeypatch.setattr(mc, "_is_vision_model_uncached", _probe)

        assert mc.is_vision_model("some/vlm", local_files_only = True) is False
        assert mc.is_vision_model("some/vlm", local_files_only = False) is True
        assert seen == [True, False]
        assert mc.is_vision_model("some/vlm", local_files_only = False) is True
        assert seen == [True, False]
        mc._vision_detection_cache.clear()


import json as _json

from utils.models.model_config import (
    _AUDIO_ONLY_MODEL_TYPES,
    _VISION_CHECK_INLINE_HELPERS,
    _VISION_CHECK_SCRIPT,
    _is_vlm,
    _raw_config_has_vision_config,
)


def _write_config(tmp_path, config):
    (tmp_path / "config.json").write_text(_json.dumps(config))
    return tmp_path


class TestRawConfigVlmDetection:
    """Exercises the real _raw_config_has_vision_config, which the cache tests above mock out."""

    def test_truthy_vision_config(self, tmp_path):
        p = _write_config(tmp_path, {"vision_config": {"hidden_size": 1024}})
        assert _raw_config_has_vision_config(str(p)) is True

    def test_empty_vision_config_key(self, tmp_path):
        p = _write_config(tmp_path, {"vision_config": {}})
        assert _raw_config_has_vision_config(str(p)) is True

    def test_arch_suffix_detection(self, tmp_path):
        p = _write_config(
            tmp_path,
            {
                "architectures": ["Gemma4ForConditionalGeneration"],
                "model_type": "gemma4",
            },
        )
        assert _raw_config_has_vision_config(str(p)) is True

    def test_img_processor_key(self, tmp_path):
        p = _write_config(tmp_path, {"img_processor": {"image_size": 336}})
        assert _raw_config_has_vision_config(str(p)) is True

    def test_image_token_index_key(self, tmp_path):
        p = _write_config(tmp_path, {"image_token_index": 32000})
        assert _raw_config_has_vision_config(str(p)) is True

    def test_known_vlm_model_type(self, tmp_path):
        p = _write_config(tmp_path, {"model_type": "gemma4"})
        assert _raw_config_has_vision_config(str(p)) is True

    def test_plain_text_model_returns_false(self, tmp_path):
        p = _write_config(
            tmp_path,
            {"model_type": "llama", "architectures": ["LlamaForCausalLM"]},
        )
        assert _raw_config_has_vision_config(str(p)) is False

    def test_missing_config_returns_none(self, tmp_path):
        assert _raw_config_has_vision_config(str(tmp_path)) is None


class TestSubprocessScript:
    def test_does_not_import_parent_module(self):
        assert "from utils.models.model_config" not in _VISION_CHECK_SCRIPT

    def test_inline_is_vlm_executes_correctly(self):
        ns: dict = {}
        exec(_VISION_CHECK_INLINE_HELPERS, ns)
        inline_is_vlm = ns["_is_vlm"]

        class _C:
            def __init__(self, **kw):
                for k, v in kw.items():
                    setattr(self, k, v)

        assert (
            inline_is_vlm(
                _C(
                    model_type = "gemma4",
                    architectures = ["Gemma4ForConditionalGeneration"],
                )
            )
            is True
        )
        assert (
            inline_is_vlm(_C(model_type = "gemma4_text", architectures = ["Gemma4ForCausalLM"]))
            is False
        )
        assert inline_is_vlm(_C(model_type = "llama", architectures = ["LlamaForCausalLM"])) is False


class TestVlmAudioExclusion:
    """The {csm, whisper} audio guard applies inside _is_vlm and the raw and subprocess fallbacks too."""

    def test_audio_only_set_canonical(self):
        assert {"csm", "whisper"} <= _AUDIO_ONLY_MODEL_TYPES

    def test_is_vlm_excludes_whisper(self):
        cfg = MagicMock(spec = [])
        cfg.model_type = "whisper"
        cfg.architectures = ["WhisperForConditionalGeneration"]
        assert _is_vlm(cfg) is False

    def test_raw_fallback_excludes_whisper(self, tmp_path):
        p = _write_config(
            tmp_path,
            {
                "architectures": ["WhisperForConditionalGeneration"],
                "model_type": "whisper",
            },
        )
        assert _raw_config_has_vision_config(str(p)) is False

    def test_inline_subprocess_helper_excludes_whisper(self):
        ns: dict = {}
        exec(_VISION_CHECK_INLINE_HELPERS, ns)
        cfg = MagicMock(spec = [])
        cfg.model_type = "whisper"
        cfg.architectures = ["WhisperForConditionalGeneration"]
        assert ns["_is_vlm"](cfg) is False

    @patch("utils.models.model_config._is_vision_model_subprocess", return_value = None)
    @patch("utils.transformers_version.needs_transformers_5", return_value = True)
    def test_t5_subprocess_none_falls_back_through_raw_for_whisper(
        self, mock_needs_t5, mock_subprocess, tmp_path
    ):
        _write_config(
            tmp_path,
            {
                "architectures": ["WhisperForConditionalGeneration"],
                "model_type": "whisper",
            },
        )
        assert is_vision_model(str(tmp_path)) is False


class TestAudioDetectionCacheTokenAware:
    """The audio cache mirrors the vision cache: keyed by (model, token_fingerprint)
    so an unauthenticated miss cannot poison a later authenticated lookup."""

    def test_audio_cache_is_token_aware(self, monkeypatch):
        import utils.models.model_config as mc

        mc._audio_detection_cache.clear()
        calls = []

        def _fake(
            name,
            hf_token = None,
            local_files_only = False,
        ):
            calls.append(hf_token)
            return ("bicodec", True) if hf_token else (None, True)

        monkeypatch.setattr(mc, "_detect_audio_from_tokenizer", _fake)
        monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
        monkeypatch.setattr(mc, "resolve_cached_repo_id_case", lambda n, *_a, **_k: n)
        monkeypatch.setattr(mc, "_env_offline", lambda: False)

        assert mc.detect_audio_type("private/spark") is None
        assert mc.detect_audio_type("private/spark", hf_token = "hf_x") == "bicodec"
        assert calls == [None, "hf_x"]

        assert mc.detect_audio_type("private/spark", hf_token = "hf_x") == "bicodec"
        assert calls == [None, "hf_x"]
        mc._audio_detection_cache.clear()

    def test_audio_cache_is_revision_aware(self, monkeypatch):
        import utils.models.model_config as mc

        mc._audio_detection_cache.clear()
        calls = []

        def _fake(
            name,
            hf_token = None,
            local_files_only = False,
            revision = None,
        ):
            calls.append(revision)
            return ("csm", True) if revision == "commit-a" else (None, True)

        monkeypatch.setattr(mc, "_detect_audio_from_tokenizer", _fake)
        monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
        monkeypatch.setattr(mc, "resolve_cached_repo_id_case", lambda n, *_a, **_k: n)
        monkeypatch.setattr(mc, "_env_offline", lambda: False)

        assert mc.detect_audio_type("org/model", revision = "commit-a") == "csm"
        assert mc.detect_audio_type("org/model", revision = "commit-b") is None
        assert mc.detect_audio_type("org/model", revision = "commit-a") == "csm"
        assert calls == ["commit-a", "commit-b"]
        assert mc._audio_detection_cache[("org/model", None, False, "commit-a")] == "csm"
        assert mc._audio_detection_cache[("org/model", None, False, "commit-b")] is None
        mc._audio_detection_cache.clear()

    def test_transient_none_is_not_cached_but_definitive_none_is(self, monkeypatch):
        """Transient probe errors (definitive=False) are retried; a definitive miss is cached."""
        mc = _shared_setup_1(monkeypatch)

        transient_calls = []

        def _transient(
            name,
            hf_token = None,
            local_files_only = False,
        ):
            transient_calls.append(hf_token)
            return (None, False)

        monkeypatch.setattr(mc, "_detect_audio_from_tokenizer", _transient)
        assert mc.detect_audio_type("flaky/model") is None
        assert mc.detect_audio_type("flaky/model") is None
        assert transient_calls == [None, None]

        definitive_calls = []

        def _definitive(
            name,
            hf_token = None,
            local_files_only = False,
        ):
            definitive_calls.append(hf_token)
            return (None, True)

        monkeypatch.setattr(mc, "_detect_audio_from_tokenizer", _definitive)
        assert mc.detect_audio_type("plain/text-model") is None
        assert mc.detect_audio_type("plain/text-model") is None
        assert definitive_calls == [None]
        mc._audio_detection_cache.clear()

    def test_local_only_negative_does_not_poison_online(self, monkeypatch):
        """An offline negative must not poison a later online probe, or audio is sent to text."""
        mc = _shared_setup_1(monkeypatch)
        # Pin env-offline off so the key tracks the kwarg.
        monkeypatch.setattr(mc, "_env_offline", lambda: False)

        seen = []

        def _probe(
            name,
            hf_token = None,
            local_files_only = False,
        ):
            seen.append(local_files_only)
            return (None, True) if local_files_only else ("snac", True)

        monkeypatch.setattr(mc, "_detect_audio_from_tokenizer", _probe)

        assert mc.detect_audio_type("some/audio-model", local_files_only = True) is None
        assert mc.detect_audio_type("some/audio-model", local_files_only = False) == "snac"
        assert seen == [True, False]
        assert mc.detect_audio_type("some/audio-model", local_files_only = False) == "snac"
        assert seen == [True, False]
        mc._audio_detection_cache.clear()

    def test_env_offline_negative_does_not_poison_online(self, monkeypatch):
        """An env-offline probe (default local_files_only=False) must cache under the
        effective-offline key, so clearing the env var later doesn't leak a stale negative."""
        mc = _shared_setup_1(monkeypatch)

        env_offline = {"v": True}
        monkeypatch.setattr(mc, "_env_offline", lambda: env_offline["v"])

        seen = []

        def _probe(
            name,
            hf_token = None,
            local_files_only = False,
        ):
            seen.append(local_files_only)
            return (None, True) if local_files_only else ("snac", True)

        monkeypatch.setattr(mc, "_detect_audio_from_tokenizer", _probe)

        assert mc.detect_audio_type("some/audio-model") is None
        assert seen == [True]
        env_offline["v"] = False
        assert mc.detect_audio_type("some/audio-model") == "snac"
        assert seen == [True, False]
        mc._audio_detection_cache.clear()


class TestRevisionAwareAudioReads:
    @staticmethod
    def _tokenizer_config(*tokens):
        return {
            "added_tokens_decoder": {
                str(index): {"content": token} for index, token in enumerate(tokens)
            }
        }

    def test_local_cache_reads_only_requested_snapshot(self, monkeypatch, tmp_path):
        import utils.models.model_config as mc

        repo_dir = tmp_path / "models--org--model"
        commit_a = repo_dir / "snapshots" / "commit-a"
        commit_b = repo_dir / "snapshots" / "commit-b"
        commit_a.mkdir(parents = True)
        commit_b.mkdir(parents = True)
        (commit_a / "tokenizer_config.json").write_text(
            _json.dumps(self._tokenizer_config("<|AUDIO|>", "<|audio_eos|>"))
        )
        (commit_b / "tokenizer_config.json").write_text(
            _json.dumps(self._tokenizer_config("<ordinary-token>"))
        )

        monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
        monkeypatch.setattr(mc, "get_cache_path", lambda *_a, **_k: repo_dir)
        monkeypatch.setattr(mc, "_env_offline", lambda: True)

        assert _detect_audio_from_tokenizer(
            "org/model",
            local_files_only = True,
            revision = "commit-b",
        ) == (None, True)
        assert _detect_audio_from_tokenizer(
            "org/model",
            local_files_only = True,
            revision = "commit-a",
        ) == ("csm", True)

    def test_remote_tokenizer_read_uses_requested_revision(self, monkeypatch):
        import requests
        import utils.models.model_config as mc

        response = MagicMock(status_code = 200, ok = True)
        response.json.return_value = self._tokenizer_config(
            "<|AUDIO|>",
            "<|audio_eos|>",
        )
        get = MagicMock(return_value = response)
        monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
        monkeypatch.setattr(mc, "get_cache_path", lambda *_a, **_k: None)
        monkeypatch.setattr(mc, "_env_offline", lambda: False)
        monkeypatch.setattr(requests, "get", get)

        assert _detect_audio_from_tokenizer("org/model", revision = "refs/pr/7") == ("csm", True)
        assert get.call_args.args[0] == (
            "https://huggingface.co/org/model/resolve/refs%2Fpr%2F7/tokenizer_config.json"
        )

    def test_remote_tokenizer_read_keeps_main_without_revision(self, monkeypatch):
        import requests
        import utils.models.model_config as mc

        response = MagicMock(status_code = 200, ok = True)
        response.json.return_value = self._tokenizer_config("<|startoftranscript|>")
        get = MagicMock(return_value = response)
        monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
        monkeypatch.setattr(mc, "get_cache_path", lambda *_a, **_k: None)
        monkeypatch.setattr(mc, "_env_offline", lambda: False)
        monkeypatch.setattr(requests, "get", get)

        assert _detect_audio_from_tokenizer("org/model") == ("whisper", True)
        assert get.call_args.args[0] == (
            "https://huggingface.co/org/model/resolve/main/tokenizer_config.json"
        )


class TestEnvOfflineParsing:
    """_env_offline accepts the canonical truthy set (strip+lower, on/true/yes/1); it gates
    the requests.get fallback and the cache keys, so 'on' or ' 1 ' must still count as offline."""

    def test_truthy_values_recognized(self, monkeypatch):
        import utils.models.model_config as mc
        for var in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"):
            for val in ("1", "true", "TRUE", "yes", "Yes", "on", "ON", " 1 ", " on ", "\ttrue\n"):
                monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
                monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False)
                monkeypatch.setenv(var, val)
                assert mc._env_offline() is True, f"{var}={val!r} should be offline"

    def test_falsy_values_not_offline(self, monkeypatch):
        import utils.models.model_config as mc

        monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
        monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False)
        assert mc._env_offline() is False
        for val in ("", "0", "false", "no", "off", "2", "onn"):
            monkeypatch.setenv("HF_HUB_OFFLINE", val)
            assert mc._env_offline() is False, f"HF_HUB_OFFLINE={val!r} should not be offline"


def _hub_cached_repo(
    tmp_path,
    repo_id,
    files,
    sha = "abc123",
):
    """A repo laid out the way the HF hub cache lays one out."""
    repo_dir = tmp_path / ("models--" + repo_id.replace("/", "--"))
    snapshot = repo_dir / "snapshots" / sha
    snapshot.mkdir(parents = True)
    for name, text in files.items():
        (snapshot / name).write_text(text, encoding = "utf-8")
    return repo_dir, snapshot


def _probe_against_cache(
    monkeypatch,
    repo_dir,
    *,
    sha = "abc123",
    listed = ("config.json",),
    remote_config = None,
    **kwargs,
):
    """Runs the vision probe on a cached snapshot and logs Hub reads; listed=None means no document."""
    import huggingface_hub
    import utils.hf_probe as hf_probe
    import utils.models.model_config as mc

    reads: list = []
    monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
    monkeypatch.setattr(mc, "get_cache_path", lambda *_a, **_k: repo_dir)

    def _info(
        model_name,
        hf_token = None,
        **kw,
    ):
        if listed is None:
            raise ConnectionError("no repo document")
        return _types.SimpleNamespace(
            sha = sha,
            siblings = [_types.SimpleNamespace(rfilename = name) for name in listed],
        )

    monkeypatch.setattr(mc, "_hub_model_info", _info)

    def _absent(model_name, filename, **kw):
        reads.append(("absent", filename))
        return remote_config is None

    def _download(**kw):
        reads.append(("download", kw.get("filename")))
        path = repo_dir / "remote-config.json"
        path.write_text(_json.dumps(remote_config), encoding = "utf-8")
        return str(path)

    monkeypatch.setattr(hf_probe, "hf_file_definitely_absent", _absent)
    monkeypatch.setattr(huggingface_hub, "hf_hub_download", _download)

    return mc._raw_config_has_vision_config("acme/vlm", **kwargs), reads


@pytest.mark.parametrize(
    "cached, expected",
    [({"vision_config": {"hidden_size": 8}}, True), ({"model_type": "llama"}, False)],
)
def test_the_current_snapshot_answers_without_fetching_the_file(
    tmp_path, monkeypatch, cached, expected
):
    """The cached snapshot answers without fetching config.json; the repo document is still read."""
    repo_dir, _ = _hub_cached_repo(tmp_path, "acme/vlm", {"config.json": _json.dumps(cached)})

    answer, reads = _probe_against_cache(monkeypatch, repo_dir)

    assert answer is expected
    assert reads == []


@pytest.mark.parametrize(
    "case, files, kwargs",
    [
        # A repo re-downloaded at a new commit keeps the old snapshot beside the new one.
        ("stale snapshot", {"config.json": '{"model_type": "llama"}'}, {"sha": "new"}),
        ("file not downloaded", {"modules.json": "[]"}, {}),
        # Offline, or on any failed read, there is nothing to judge the snapshot against.
        ("no repo document", {"config.json": '{"model_type": "llama"}'}, {"listed": None}),
        # The cached snapshot is whichever was downloaded, not the revision asked for.
        ("pinned revision", {"config.json": '{"model_type": "llama"}'}, {"revision": "commit-a"}),
        # Reading the cache authorizes nothing, so a caller who forced anonymity keeps it.
        ("anonymous", {"config.json": '{"model_type": "llama"}'}, {"hf_token": False}),
    ],
)
def test_the_hub_still_answers_when_the_snapshot_may_not(
    tmp_path, monkeypatch, case, files, kwargs
):
    repo_dir, _ = _hub_cached_repo(tmp_path, "acme/vlm", files)

    answer, reads = _probe_against_cache(
        monkeypatch, repo_dir, remote_config = {"vision_config": {"hidden_size": 8}}, **kwargs
    )

    assert answer is True, case
    assert reads == [("absent", "config.json"), ("download", "config.json")], case


def test_local_files_only_reads_no_repo_document(tmp_path, monkeypatch):
    """The caller asked for no network; resolving the current commit would be one."""
    import utils.models.model_config as mc

    repo_dir, _ = _hub_cached_repo(
        tmp_path, "acme/vlm", {"config.json": _json.dumps({"model_type": "llama"})}
    )
    reads: list = []

    def _info(
        model_name,
        hf_token = None,
        **kw,
    ):
        reads.append(model_name)
        raise AssertionError("the repo document must not be read")

    monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
    monkeypatch.setattr(mc, "get_cache_path", lambda *_a, **_k: repo_dir)
    monkeypatch.setattr(mc, "_hub_model_info", _info)

    import huggingface_hub

    downloads: list = []

    def _download(**kw):
        downloads.append(kw.get("local_files_only"))
        return str(repo_dir / "snapshots" / "abc123" / "config.json")

    monkeypatch.setattr(huggingface_hub, "hf_hub_download", _download)

    assert mc._raw_config_has_vision_config("acme/vlm", local_files_only = True) is False
    assert reads == []
    assert downloads == [True]


def test_nothing_cached_costs_no_repo_document(tmp_path, monkeypatch):
    """The document is only worth a round trip when there is a snapshot to judge with it."""
    import utils.models.model_config as mc

    empty = tmp_path / "models--acme--vlm"
    empty.mkdir()
    monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
    monkeypatch.setattr(mc, "get_cache_path", lambda *_a, **_k: empty)

    reads: list = []

    def _info(
        model_name,
        hf_token = None,
        **kw,
    ):
        reads.append(model_name)
        raise AssertionError("the repo document must not be read")

    monkeypatch.setattr(mc, "_hub_model_info", _info)

    assert mc._current_cached_snapshot("acme/vlm") is None
    assert reads == []


def test_the_current_snapshot_is_the_one_the_repo_document_names(tmp_path, monkeypatch):
    """The helper both probes rest on, exercised rather than stubbed."""
    import utils.models.model_config as mc

    repo_dir, snapshot = _hub_cached_repo(tmp_path, "acme/vlm", {"config.json": "{}"}, sha = "aaa")
    (repo_dir / "snapshots" / "bbb").mkdir()
    monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
    monkeypatch.setattr(mc, "get_cache_path", lambda *_a, **_k: repo_dir)

    def _info(
        model_name,
        hf_token = None,
        **kw,
    ):
        return _types.SimpleNamespace(
            sha = "aaa", siblings = [_types.SimpleNamespace(rfilename = "config.json")]
        )

    monkeypatch.setattr(mc, "_hub_model_info", _info)

    assert mc._current_cached_snapshot("acme/vlm") == (snapshot, {"config.json"})
    assert mc._current_cached_snapshot("acme/vlm", False) is None


def test_a_repo_that_publishes_no_config_is_settled_by_the_document(tmp_path, monkeypatch):
    """A document that omits config.json already settles it; probing the Hub for it wastes a round trip."""
    repo_dir, _ = _hub_cached_repo(tmp_path, "acme/vlm", {"model-Q4_K_M.gguf": "x"})

    result, reads = _probe_against_cache(monkeypatch, repo_dir, listed = ("model-Q4_K_M.gguf",))

    assert result is None
    assert reads == []


def test_a_config_the_repo_has_but_the_cache_lacks_still_asks(tmp_path, monkeypatch):
    """Only a document omitting the file settles it; one listing an absent file means fetch it."""
    repo_dir, _ = _hub_cached_repo(tmp_path, "acme/vlm", {"model-Q4_K_M.gguf": "x"})

    result, reads = _probe_against_cache(
        monkeypatch,
        repo_dir,
        listed = ("model-Q4_K_M.gguf", "config.json"),
        remote_config = {"vision_config": {"hidden_size": 8}},
    )

    assert result is True
    assert reads == [("absent", "config.json"), ("download", "config.json")]