Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""is_vision_model() caches True, False, subprocess and exception-fallback results per model."""
import struct
import sys
import types as _types
from pathlib import Path
from unittest.mock import patch, MagicMock
import pytest
def _shared_setup_1(monkeypatch):
import utils.models.model_config as mc
mc._audio_detection_cache.clear()
monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
monkeypatch.setattr(mc, "resolve_cached_repo_id_case", lambda n, *_a, **_k: n)
return mc
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
_loggers_stub = _types.ModuleType("loggers")
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
sys.modules.setdefault("loggers", _loggers_stub)
from utils.models.model_config import (
ModelConfig,
load_model_config,
is_vision_model,
_detect_audio_from_tokenizer,
_is_vision_model_subprocess,
_is_vision_model_uncached,
_vision_detection_cache,
)
@pytest.fixture(autouse = True)
def _clear_vision_cache(tmp_path, monkeypatch):
"""Run each test from an empty cwd: a relative model id that exists on disk counts as local."""
monkeypatch.chdir(tmp_path)
_vision_detection_cache.clear()
yield
_vision_detection_cache.clear()
class TestVisionCacheHitMiss:
"""Verify the cache prevents redundant detection calls."""
@patch("utils.models.model_config._is_vision_model_uncached", return_value = True)
def test_second_call_uses_cache(self, mock_uncached):
"""Two calls for the same model invoke the uncached fn once."""
assert is_vision_model("org/my-vlm") is True
assert is_vision_model("org/my-vlm") is True
mock_uncached.assert_called_once_with("org/my-vlm", None, local_files_only = False)
@patch("utils.models.model_config._is_vision_model_uncached", return_value = False)
def test_different_models_each_detected(self, mock_uncached):
"""Different model names should each trigger detection."""
is_vision_model("model-a")
is_vision_model("model-b")
assert mock_uncached.call_count == 2
@patch("utils.models.model_config._is_vision_model_uncached", return_value = True)
def test_cache_returns_correct_value(self, mock_uncached):
"""The cached value must match what _is_vision_model_uncached returned."""
first = is_vision_model("org/vlm")
second = is_vision_model("org/vlm")
assert first is True
assert second is True
class TestVisionCacheStoresFalse:
"""Non-VLM results (False) must also be cached to avoid re-detection."""
@patch("utils.models.model_config._is_vision_model_uncached", return_value = False)
def test_false_result_cached(self, mock_uncached):
assert is_vision_model("org/text-only") is False
assert is_vision_model("org/text-only") is False
mock_uncached.assert_called_once()
assert _vision_detection_cache[("org/text-only", None, False)] is False
class TestVisionCacheSubprocessPath:
"""transformers 5.x models go through _is_vision_model_subprocess, which the cache spawns at
most once per model per process."""
@patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
@patch("utils.models.model_config._is_vision_model_subprocess", return_value = True)
@patch("utils.transformers_version.needs_transformers_5", return_value = True)
def test_subprocess_called_once_with_cache(self, mock_needs_t5, mock_subprocess, mock_raw):
"""An inconclusive raw config runs the 5.x subprocess once; the second call is cached."""
assert is_vision_model("unsloth/Qwen3.5-2B") is True
assert is_vision_model("unsloth/Qwen3.5-2B") is True
mock_subprocess.assert_called_once()
assert _vision_detection_cache[("unsloth/Qwen3.5-2B", None, False)] is True
@patch("utils.models.model_config._raw_config_has_vision_config", return_value = True)
@patch("utils.models.model_config._is_vision_model_subprocess", return_value = None)
@patch("utils.transformers_version.needs_transformers_5", return_value = True)
def test_raw_config_primary_skips_subprocess(
self, mock_needs_t5, mock_subprocess, mock_raw_config
):
# The raw config.json read is the primary path; a definitive answer there never reaches the subprocess.
assert is_vision_model("unsloth/gemma-4-E4B-it") is True
assert is_vision_model("unsloth/gemma-4-E4B-it") is True
mock_raw_config.assert_called_once_with(
"unsloth/gemma-4-E4B-it", hf_token = None, local_files_only = False
)
mock_subprocess.assert_not_called()
def _projector_declaring(path: Path, key: str) -> Path:
"""A minimal GGUF carrying one ``clip.has_*_encoder`` bool, no tensors."""
kv = struct.pack("<Q", len(key)) + key.encode() + struct.pack("<I", 7) + struct.pack("<?", True)
path.write_bytes(struct.pack("<IIQQ", 0x46554747, 3, 0, 1) + kv)
return path
class TestLocalGgufVisionDetection:
"""Every projector fixture is non-empty, since ``detect_mmproj_file`` skips a zero-byte one
as an interrupted download; those built by ``_projector_declaring`` also carry a header,
because the capability they assert is read from it."""
@patch(
"utils.models.model_config._is_vision_model_subprocess",
side_effect = AssertionError("GGUF must not use Transformers vision detection"),
)
def test_qwen36_gguf_with_mmproj_skips_transformers(self, mock_subprocess, tmp_path):
model = tmp_path / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
model.write_bytes(b"")
(tmp_path / "mmproj-F32.gguf").write_bytes(b"\0" * 32)
assert is_vision_model(str(model)) is True
mock_subprocess.assert_not_called()
@patch(
"utils.models.model_config._is_vision_model_subprocess",
side_effect = AssertionError("GGUF must not use Transformers vision detection"),
)
def test_direct_gguf_in_variant_subdir_finds_snapshot_mmproj(self, mock_subprocess, tmp_path):
variant_dir = tmp_path / "BF16"
variant_dir.mkdir()
model = variant_dir / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
model.write_bytes(b"")
(tmp_path / "mmproj-F32.gguf").write_bytes(b"\0" * 32)
assert is_vision_model(str(model)) is True
mock_subprocess.assert_not_called()
@patch(
"utils.models.model_config._is_vision_model_subprocess",
side_effect = AssertionError("GGUF must not use Transformers vision detection"),
)
def test_qwen36_gguf_without_mmproj_skips_transformers(self, mock_subprocess, tmp_path):
model = tmp_path / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
model.write_bytes(b"")
assert is_vision_model(str(model)) is False
mock_subprocess.assert_not_called()
def test_local_gguf_check_observes_mmproj_added_later(self, tmp_path):
model = tmp_path / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
model.write_bytes(b"")
assert is_vision_model(str(model)) is False
(tmp_path / "mmproj-F32.gguf").write_bytes(b"\0" * 32)
assert is_vision_model(str(model)) is True
@patch(
"utils.models.model_config._is_vision_model_subprocess",
side_effect = AssertionError("GGUF must not use Transformers vision detection"),
)
def test_ui_selection_returns_local_gguf_config(self, mock_subprocess, tmp_path):
model = tmp_path / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
model.write_bytes(b"")
mmproj = tmp_path / "mmproj-F32.gguf"
mmproj.write_bytes(b"\0" * 32)
config = ModelConfig.from_ui_selection(str(model), None)
assert config is not None
assert config.is_gguf is True
assert config.is_vision is True
assert config.gguf_mmproj_file == str(mmproj.resolve())
mock_subprocess.assert_not_called()
@patch(
"utils.models.model_config._is_vision_model_subprocess",
side_effect = AssertionError("GGUF must not use Transformers vision detection"),
)
def test_ui_selection_direct_gguf_in_variant_subdir_keeps_mmproj(
self, mock_subprocess, tmp_path
):
variant_dir = tmp_path / "BF16"
variant_dir.mkdir()
model = variant_dir / "Qwen3.6-27B-UD-Q4_K_XL-MTP.gguf"
model.write_bytes(b"")
mmproj = tmp_path / "mmproj-F32.gguf"
mmproj.write_bytes(b"\0" * 32)
config = ModelConfig.from_ui_selection(str(model), None)
assert config is not None
assert config.is_gguf is True
assert config.is_vision is True
assert config.gguf_mmproj_file == str(mmproj.resolve())
mock_subprocess.assert_not_called()
def test_an_audio_only_projector_is_not_a_vision_model(self, tmp_path):
"""ultravox, Voxtral and Qwen3-ASR take audio projectors only, so no image input."""
model = tmp_path / "Voxtral-Mini-3B-2507-Q4_K_M.gguf"
model.write_bytes(b"\0" * 32)
_projector_declaring(tmp_path / "mmproj-F16.gguf", "clip.has_audio_encoder")
assert is_vision_model(str(model)) is False
def test_a_projector_declaring_vision_is_still_a_vision_model(self, tmp_path):
model = tmp_path / "Qwen3-VL-8B-Instruct-Q4_K_M.gguf"
model.write_bytes(b"\0" * 32)
_projector_declaring(tmp_path / "mmproj-F16.gguf", "clip.has_vision_encoder")
assert is_vision_model(str(model)) is True
@patch(
"utils.models.model_config._is_vision_model_subprocess",
side_effect = AssertionError("GGUF must not use Transformers vision detection"),
)
def test_named_quant_in_a_subdir_reads_the_snapshot_projector(self, mock_subprocess, tmp_path):
"""A repo whose quants all live under a per-quant subdir has no weight file at the
snapshot root, which is the only place the root-level detector looks (#8772)."""
variant_dir = tmp_path / "UD-Q4_K_XL"
variant_dir.mkdir()
(variant_dir / "Qwen3-VL-235B-UD-Q4_K_XL-00001-of-00002.gguf").write_bytes(b"\0" * 32)
(variant_dir / "Qwen3-VL-235B-UD-Q4_K_XL-00002-of-00002.gguf").write_bytes(b"\0" * 32)
(tmp_path / "mmproj-F32.gguf").write_bytes(b"\0" * 32)
assert is_vision_model(str(tmp_path), gguf_variant = "UD-Q4_K_XL") is True
mock_subprocess.assert_not_called()
@patch(
"utils.models.model_config._is_vision_model_subprocess",
side_effect = AssertionError("GGUF must not use Transformers vision detection"),
)
def test_named_quant_in_a_subdir_without_a_projector_is_text_only(
self, mock_subprocess, tmp_path
):
variant_dir = tmp_path / "UD-Q4_K_XL"
variant_dir.mkdir()
(variant_dir / "Qwen3-235B-UD-Q4_K_XL.gguf").write_bytes(b"\0" * 32)
assert is_vision_model(str(tmp_path), gguf_variant = "UD-Q4_K_XL") is False
mock_subprocess.assert_not_called()
@pytest.mark.parametrize(
"variant, expected",
[("Q4_K_M", True), ("Q8_0", False)],
)
def test_each_quant_answers_what_a_load_of_that_quant_would_see(
self, tmp_path, variant, expected
):
"""One quant keeps the projector beside it and the other does not, so a probe that
reads any quant of the directory answers one of them wrongly."""
variant_dir = tmp_path / "Q4_K_M"
variant_dir.mkdir()
(variant_dir / "Qwen3-VL-8B-Instruct-Q4_K_M.gguf").write_bytes(b"\0" * 32)
(variant_dir / "mmproj-F16.gguf").write_bytes(b"\0" * 32)
(tmp_path / "Qwen3-VL-8B-Instruct-Q8_0.gguf").write_bytes(b"\0" * 32)
config = ModelConfig.from_identifier(str(tmp_path), gguf_variant = variant)
assert config is not None
assert config.is_vision is expected
assert is_vision_model(str(tmp_path), gguf_variant = variant) is expected
@patch("utils.models.model_config._is_vision_model_uncached", return_value = False)
def test_a_quant_that_is_not_on_disk_is_not_answered_by_another_one(
self, mock_uncached, tmp_path
):
"""A load of an absent quant resolves no GGUF at all, so neither may the probe: the
projector beside the quant that IS on disk says nothing about the one asked for."""
(tmp_path / "Qwen3-VL-8B-Instruct-Q8_0.gguf").write_bytes(b"\0" * 32)
(tmp_path / "mmproj-F16.gguf").write_bytes(b"\0" * 32)
config = ModelConfig.from_identifier(str(tmp_path), gguf_variant = "UD-Q4_K_XL")
assert config is not None
assert config.is_gguf is False
assert is_vision_model(str(tmp_path), gguf_variant = "UD-Q4_K_XL") is False
class TestVisionCacheOnException:
"""On exception, _is_vision_model_uncached distinguishes permanent
failures (cached as False) from transient ones (returned as None,
not cached, so the next call retries). Verify both contracts."""
@patch(
"utils.models.model_config.load_model_config",
side_effect = ValueError("bad config"),
)
@patch("utils.transformers_version.needs_transformers_5", return_value = False)
def test_permanent_exception_result_cached(self, mock_needs_t5, mock_load_config):
"""Permanent failures (e.g. ValueError, GatedRepoError) cache False so later calls do not retry."""
assert is_vision_model("broken/model") is False
assert is_vision_model("broken/model") is False
mock_load_config.assert_called_once()
@patch(
"utils.models.model_config.load_model_config",
side_effect = OSError("network down"),
)
@patch("utils.transformers_version.needs_transformers_5", return_value = False)
def test_transient_exception_not_cached(self, mock_needs_t5, mock_load_config):
"""A transient failure (OSError, timeouts) returns None from
_is_vision_model_uncached, surfaces as False, and is NOT cached
so the next call retries."""
assert is_vision_model("broken/model") is False
assert is_vision_model("broken/model") is False
assert mock_load_config.call_count == 2
class TestVisionCacheDirectPath:
"""Non-5.x models detect via load_model_config; the cache must handle them the same way."""
@patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
@patch("utils.transformers_version.needs_transformers_5", return_value = False)
@patch("utils.models.model_config.load_model_config")
def test_direct_vlm_detection_cached(self, mock_load_config, mock_needs_t5, mock_raw):
"""A standard VLM detected via architecture suffix should be cached."""
cfg = MagicMock(spec = [])
cfg.model_type = "gemma3"
cfg.architectures = ["Gemma3ForConditionalGeneration"]
mock_load_config.return_value = cfg
assert is_vision_model("google/gemma-3-4b-it") is True
assert is_vision_model("google/gemma-3-4b-it") is True
mock_load_config.assert_called_once()
@patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
@patch("utils.transformers_version.needs_transformers_5", return_value = False)
@patch("utils.models.model_config.load_model_config")
def test_direct_non_vlm_detection_cached(self, mock_load_config, mock_needs_t5, mock_raw):
"""A standard text model (no VLM indicators) should cache False."""
cfg = MagicMock(spec = [])
cfg.model_type = "llama"
cfg.architectures = ["LlamaForCausalLM"]
mock_load_config.return_value = cfg
assert is_vision_model("meta-llama/Llama-3-8B") is False
assert is_vision_model("meta-llama/Llama-3-8B") is False
mock_load_config.assert_called_once()
@patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
@patch("utils.transformers_version.needs_transformers_5", return_value = False)
@patch("utils.models.model_config.load_model_config")
def test_vision_config_attr_detected_and_cached(
self, mock_load_config, mock_needs_t5, mock_raw
):
"""Models with vision_config (LLaVA, Qwen2-VL, etc.) should be cached as True."""
cfg = MagicMock(spec = [])
cfg.model_type = "qwen2_vl"
cfg.architectures = ["Qwen2VLForCausalLM"]
cfg.vision_config = {"hidden_size": 1024}
mock_load_config.return_value = cfg
assert is_vision_model("Qwen/Qwen2-VL-7B") is True
assert is_vision_model("Qwen/Qwen2-VL-7B") is True
mock_load_config.assert_called_once()
@patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
@patch("utils.transformers_version.needs_transformers_5", return_value = False)
@patch("utils.models.model_config.load_model_config")
def test_gemma4_model_type_detected_and_cached(self, mock_load_config, mock_needs_t5, mock_raw):
cfg = MagicMock(spec = [])
cfg.model_type = "gemma4"
cfg.architectures = ["Gemma4ForConditionalGeneration"]
mock_load_config.return_value = cfg
assert is_vision_model("google/gemma-4-E4B-it") is True
assert is_vision_model("google/gemma-4-E4B-it") is True
mock_load_config.assert_called_once()
@patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
@patch("utils.transformers_version.needs_transformers_5", return_value = False)
@patch("utils.models.model_config.load_model_config")
def test_gemma4_audio_subconfig_not_detected_as_vision(
self, mock_load_config, mock_needs_t5, mock_raw
):
cfg = MagicMock(spec = [])
cfg.model_type = "gemma4_audio"
cfg.architectures = ["Gemma4AudioModel"]
mock_load_config.return_value = cfg
assert is_vision_model("local/gemma4-audio-encoder") is False
assert is_vision_model("local/gemma4-audio-encoder") is False
mock_load_config.assert_called_once()
@patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
@patch("utils.transformers_version.needs_transformers_5", return_value = False)
@patch("utils.models.model_config.load_model_config")
def test_gemma4_text_subconfig_not_detected_as_vision(
self, mock_load_config, mock_needs_t5, mock_raw
):
cfg = MagicMock(spec = [])
cfg.model_type = "gemma4_text"
cfg.architectures = ["Gemma4ForCausalLM"]
mock_load_config.return_value = cfg
assert is_vision_model("local/gemma-4-text") is False
assert is_vision_model("local/gemma-4-text") is False
mock_load_config.assert_called_once()
@patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
@patch("utils.transformers_version.needs_transformers_5", return_value = False)
@patch("utils.models.model_config.load_model_config")
def test_audio_model_excluded_and_cached(self, mock_load_config, mock_needs_t5, mock_raw):
"""Audio-only models (csm, whisper) with ForConditionalGeneration
should be excluded from VLM detection and cached as False."""
cfg = MagicMock(spec = [])
cfg.model_type = "whisper"
cfg.architectures = ["WhisperForConditionalGeneration"]
mock_load_config.return_value = cfg
assert is_vision_model("openai/whisper-large-v3") is False
assert is_vision_model("openai/whisper-large-v3") is False
mock_load_config.assert_called_once()
class TestVisionCacheTokenHandling:
"""Cache is keyed on (model_name, hf_token), so each token runs its own gated-model check."""
@patch("utils.models.model_config._is_vision_model_uncached", return_value = True)
def test_different_tokens_trigger_new_detection(self, mock_uncached):
"""An unauthenticated False for a gated model must not block the later authenticated check."""
assert is_vision_model("gated/model", hf_token = "token-a") is True
assert is_vision_model("gated/model", hf_token = "token-b") is True
assert mock_uncached.call_count == 2
@patch("utils.models.model_config._is_vision_model_uncached", return_value = True)
def test_same_token_uses_cache(self, mock_uncached):
"""Repeated calls with identical model + token should hit cache."""
assert is_vision_model("gated/model", hf_token = "token-a") is True
assert is_vision_model("gated/model", hf_token = "token-a") is True
mock_uncached.assert_called_once()
class TestRevisionAwareVisionDetection:
"""A pinned Hub commit must flow through every vision config read and cache key."""
@patch(
"utils.models.model_config._is_vision_model_uncached",
side_effect = [False, True],
)
def test_different_revisions_do_not_share_cache(self, mock_uncached, monkeypatch):
monkeypatch.setattr("utils.models.model_config._env_offline", lambda: False)
assert is_vision_model("org/model", revision = "commit-a") is False
assert is_vision_model("org/model", revision = "commit-b") is True
assert is_vision_model("org/model", revision = "commit-a") is False
assert mock_uncached.call_count == 2
mock_uncached.assert_any_call(
"org/model",
None,
local_files_only = False,
revision = "commit-a",
)
mock_uncached.assert_any_call(
"org/model",
None,
local_files_only = False,
revision = "commit-b",
)
assert _vision_detection_cache[("org/model", None, False, "commit-a")] is False
assert _vision_detection_cache[("org/model", None, False, "commit-b")] is True
@patch("transformers.AutoConfig.from_pretrained")
def test_load_model_config_forwards_only_non_null_revision(self, from_pretrained):
load_model_config("org/model", use_auth = True, revision = "commit-a")
assert from_pretrained.call_args.kwargs["revision"] == "commit-a"
load_model_config("org/model", use_auth = True)
assert "revision" not in from_pretrained.call_args.kwargs
def test_raw_config_download_uses_revision(self, monkeypatch, tmp_path):
import utils.models.model_config as mc
config_path = tmp_path / "config.json"
config_path.write_text('{"model_type": "llama"}')
download = MagicMock(return_value = str(config_path))
monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
monkeypatch.setattr("huggingface_hub.hf_hub_download", download)
assert mc._raw_config_has_vision_config("org/model", revision = "commit-a") is False
assert download.call_args.kwargs["revision"] == "commit-a"
assert mc._raw_config_has_vision_config("org/model") is False
assert "revision" not in download.call_args.kwargs
@patch("utils.models.model_config.load_model_config")
@patch("utils.transformers_version.needs_transformers_5", return_value = False)
@patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
def test_direct_config_fallback_uses_revision(self, mock_raw, mock_needs_t5, mock_load):
cfg = MagicMock(spec = [])
cfg.model_type = "llama"
cfg.architectures = ["LlamaForCausalLM"]
mock_load.return_value = cfg
assert (
_is_vision_model_uncached(
"org/model",
hf_token = "hf_x",
revision = "commit-a",
)
is False
)
mock_load.assert_called_once_with(
"org/model",
use_auth = True,
token = "hf_x",
local_files_only = False,
revision = "commit-a",
)
@patch("utils.models.model_config._is_vision_model_subprocess", return_value = True)
@patch("utils.transformers_version.needs_transformers_5", return_value = True)
@patch("utils.models.model_config._raw_config_has_vision_config", return_value = None)
def test_transformers_5_fallback_uses_revision(self, mock_raw, mock_needs_t5, mock_subprocess):
assert (
_is_vision_model_uncached(
"org/model",
hf_token = "hf_x",
revision = "commit-a",
)
is True
)
mock_subprocess.assert_called_once_with(
"org/model",
hf_token = "hf_x",
revision = "commit-a",
)
@patch("utils.transformers_version.get_transformers_tier", return_value = "default")
@patch("utils.models.model_config.subprocess.run")
def test_subprocess_command_carries_revision(self, run, mock_tier):
run.return_value = MagicMock(
returncode = 0,
stdout = '{"is_vision": false}',
stderr = "",
)
assert (
_is_vision_model_subprocess(
"org/model",
hf_token = "hf_x",
revision = "commit-a",
)
is False
)
# The token rides in the child env only; its argv slot stays, empty.
assert run.call_args.args[0][-3:] == ["org/model", "", "commit-a"]
assert run.call_args.kwargs["env"]["HF_TOKEN"] == "hf_x"
assert 'kwargs["revision"] = revision' in run.call_args.args[0][2]
class TestVisionCacheLocalOnly:
"""local_files_only is in the cache key: an offline negative must not be reused by a
later online probe (else a VLM is routed through the text loader until restart)."""
def test_local_only_negative_does_not_poison_online(self, monkeypatch):
import utils.models.model_config as mc
mc._vision_detection_cache.clear()
monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
monkeypatch.setattr(mc, "resolve_cached_repo_id_case", lambda n, *_a, **_k: n)
# Pin env-offline off so the key tracks the kwarg.
monkeypatch.setattr(mc, "_env_offline", lambda: False)
seen = []
def _probe(
name,
hf_token = None,
local_files_only = False,
):
seen.append(local_files_only)
return False if local_files_only else True
monkeypatch.setattr(mc, "_is_vision_model_uncached", _probe)
assert mc.is_vision_model("some/vlm", local_files_only = True) is False
assert mc.is_vision_model("some/vlm", local_files_only = False) is True
assert seen == [True, False]
assert mc.is_vision_model("some/vlm", local_files_only = False) is True
assert seen == [True, False]
mc._vision_detection_cache.clear()
import json as _json
from utils.models.model_config import (
_AUDIO_ONLY_MODEL_TYPES,
_VISION_CHECK_INLINE_HELPERS,
_VISION_CHECK_SCRIPT,
_is_vlm,
_raw_config_has_vision_config,
)
def _write_config(tmp_path, config):
(tmp_path / "config.json").write_text(_json.dumps(config))
return tmp_path
class TestRawConfigVlmDetection:
"""Exercises the real _raw_config_has_vision_config, which the cache tests above mock out."""
def test_truthy_vision_config(self, tmp_path):
p = _write_config(tmp_path, {"vision_config": {"hidden_size": 1024}})
assert _raw_config_has_vision_config(str(p)) is True
def test_empty_vision_config_key(self, tmp_path):
p = _write_config(tmp_path, {"vision_config": {}})
assert _raw_config_has_vision_config(str(p)) is True
def test_arch_suffix_detection(self, tmp_path):
p = _write_config(
tmp_path,
{
"architectures": ["Gemma4ForConditionalGeneration"],
"model_type": "gemma4",
},
)
assert _raw_config_has_vision_config(str(p)) is True
def test_img_processor_key(self, tmp_path):
p = _write_config(tmp_path, {"img_processor": {"image_size": 336}})
assert _raw_config_has_vision_config(str(p)) is True
def test_image_token_index_key(self, tmp_path):
p = _write_config(tmp_path, {"image_token_index": 32000})
assert _raw_config_has_vision_config(str(p)) is True
def test_known_vlm_model_type(self, tmp_path):
p = _write_config(tmp_path, {"model_type": "gemma4"})
assert _raw_config_has_vision_config(str(p)) is True
def test_plain_text_model_returns_false(self, tmp_path):
p = _write_config(
tmp_path,
{"model_type": "llama", "architectures": ["LlamaForCausalLM"]},
)
assert _raw_config_has_vision_config(str(p)) is False
def test_missing_config_returns_none(self, tmp_path):
assert _raw_config_has_vision_config(str(tmp_path)) is None
class TestSubprocessScript:
def test_does_not_import_parent_module(self):
assert "from utils.models.model_config" not in _VISION_CHECK_SCRIPT
def test_inline_is_vlm_executes_correctly(self):
ns: dict = {}
exec(_VISION_CHECK_INLINE_HELPERS, ns)
inline_is_vlm = ns["_is_vlm"]
class _C:
def __init__(self, **kw):
for k, v in kw.items():
setattr(self, k, v)
assert (
inline_is_vlm(
_C(
model_type = "gemma4",
architectures = ["Gemma4ForConditionalGeneration"],
)
)
is True
)
assert (
inline_is_vlm(_C(model_type = "gemma4_text", architectures = ["Gemma4ForCausalLM"]))
is False
)
assert inline_is_vlm(_C(model_type = "llama", architectures = ["LlamaForCausalLM"])) is False
class TestVlmAudioExclusion:
"""The {csm, whisper} audio guard applies inside _is_vlm and the raw and subprocess fallbacks too."""
def test_audio_only_set_canonical(self):
assert {"csm", "whisper"} <= _AUDIO_ONLY_MODEL_TYPES
def test_is_vlm_excludes_whisper(self):
cfg = MagicMock(spec = [])
cfg.model_type = "whisper"
cfg.architectures = ["WhisperForConditionalGeneration"]
assert _is_vlm(cfg) is False
def test_raw_fallback_excludes_whisper(self, tmp_path):
p = _write_config(
tmp_path,
{
"architectures": ["WhisperForConditionalGeneration"],
"model_type": "whisper",
},
)
assert _raw_config_has_vision_config(str(p)) is False
def test_inline_subprocess_helper_excludes_whisper(self):
ns: dict = {}
exec(_VISION_CHECK_INLINE_HELPERS, ns)
cfg = MagicMock(spec = [])
cfg.model_type = "whisper"
cfg.architectures = ["WhisperForConditionalGeneration"]
assert ns["_is_vlm"](cfg) is False
@patch("utils.models.model_config._is_vision_model_subprocess", return_value = None)
@patch("utils.transformers_version.needs_transformers_5", return_value = True)
def test_t5_subprocess_none_falls_back_through_raw_for_whisper(
self, mock_needs_t5, mock_subprocess, tmp_path
):
_write_config(
tmp_path,
{
"architectures": ["WhisperForConditionalGeneration"],
"model_type": "whisper",
},
)
assert is_vision_model(str(tmp_path)) is False
class TestAudioDetectionCacheTokenAware:
"""The audio cache mirrors the vision cache: keyed by (model, token_fingerprint)
so an unauthenticated miss cannot poison a later authenticated lookup."""
def test_audio_cache_is_token_aware(self, monkeypatch):
import utils.models.model_config as mc
mc._audio_detection_cache.clear()
calls = []
def _fake(
name,
hf_token = None,
local_files_only = False,
):
calls.append(hf_token)
return ("bicodec", True) if hf_token else (None, True)
monkeypatch.setattr(mc, "_detect_audio_from_tokenizer", _fake)
monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
monkeypatch.setattr(mc, "resolve_cached_repo_id_case", lambda n, *_a, **_k: n)
monkeypatch.setattr(mc, "_env_offline", lambda: False)
assert mc.detect_audio_type("private/spark") is None
assert mc.detect_audio_type("private/spark", hf_token = "hf_x") == "bicodec"
assert calls == [None, "hf_x"]
assert mc.detect_audio_type("private/spark", hf_token = "hf_x") == "bicodec"
assert calls == [None, "hf_x"]
mc._audio_detection_cache.clear()
def test_audio_cache_is_revision_aware(self, monkeypatch):
import utils.models.model_config as mc
mc._audio_detection_cache.clear()
calls = []
def _fake(
name,
hf_token = None,
local_files_only = False,
revision = None,
):
calls.append(revision)
return ("csm", True) if revision == "commit-a" else (None, True)
monkeypatch.setattr(mc, "_detect_audio_from_tokenizer", _fake)
monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
monkeypatch.setattr(mc, "resolve_cached_repo_id_case", lambda n, *_a, **_k: n)
monkeypatch.setattr(mc, "_env_offline", lambda: False)
assert mc.detect_audio_type("org/model", revision = "commit-a") == "csm"
assert mc.detect_audio_type("org/model", revision = "commit-b") is None
assert mc.detect_audio_type("org/model", revision = "commit-a") == "csm"
assert calls == ["commit-a", "commit-b"]
assert mc._audio_detection_cache[("org/model", None, False, "commit-a")] == "csm"
assert mc._audio_detection_cache[("org/model", None, False, "commit-b")] is None
mc._audio_detection_cache.clear()
def test_transient_none_is_not_cached_but_definitive_none_is(self, monkeypatch):
"""Transient probe errors (definitive=False) are retried; a definitive miss is cached."""
mc = _shared_setup_1(monkeypatch)
transient_calls = []
def _transient(
name,
hf_token = None,
local_files_only = False,
):
transient_calls.append(hf_token)
return (None, False)
monkeypatch.setattr(mc, "_detect_audio_from_tokenizer", _transient)
assert mc.detect_audio_type("flaky/model") is None
assert mc.detect_audio_type("flaky/model") is None
assert transient_calls == [None, None]
definitive_calls = []
def _definitive(
name,
hf_token = None,
local_files_only = False,
):
definitive_calls.append(hf_token)
return (None, True)
monkeypatch.setattr(mc, "_detect_audio_from_tokenizer", _definitive)
assert mc.detect_audio_type("plain/text-model") is None
assert mc.detect_audio_type("plain/text-model") is None
assert definitive_calls == [None]
mc._audio_detection_cache.clear()
def test_local_only_negative_does_not_poison_online(self, monkeypatch):
"""An offline negative must not poison a later online probe, or audio is sent to text."""
mc = _shared_setup_1(monkeypatch)
# Pin env-offline off so the key tracks the kwarg.
monkeypatch.setattr(mc, "_env_offline", lambda: False)
seen = []
def _probe(
name,
hf_token = None,
local_files_only = False,
):
seen.append(local_files_only)
return (None, True) if local_files_only else ("snac", True)
monkeypatch.setattr(mc, "_detect_audio_from_tokenizer", _probe)
assert mc.detect_audio_type("some/audio-model", local_files_only = True) is None
assert mc.detect_audio_type("some/audio-model", local_files_only = False) == "snac"
assert seen == [True, False]
assert mc.detect_audio_type("some/audio-model", local_files_only = False) == "snac"
assert seen == [True, False]
mc._audio_detection_cache.clear()
def test_env_offline_negative_does_not_poison_online(self, monkeypatch):
"""An env-offline probe (default local_files_only=False) must cache under the
effective-offline key, so clearing the env var later doesn't leak a stale negative."""
mc = _shared_setup_1(monkeypatch)
env_offline = {"v": True}
monkeypatch.setattr(mc, "_env_offline", lambda: env_offline["v"])
seen = []
def _probe(
name,
hf_token = None,
local_files_only = False,
):
seen.append(local_files_only)
return (None, True) if local_files_only else ("snac", True)
monkeypatch.setattr(mc, "_detect_audio_from_tokenizer", _probe)
assert mc.detect_audio_type("some/audio-model") is None
assert seen == [True]
env_offline["v"] = False
assert mc.detect_audio_type("some/audio-model") == "snac"
assert seen == [True, False]
mc._audio_detection_cache.clear()
class TestRevisionAwareAudioReads:
@staticmethod
def _tokenizer_config(*tokens):
return {
"added_tokens_decoder": {
str(index): {"content": token} for index, token in enumerate(tokens)
}
}
def test_local_cache_reads_only_requested_snapshot(self, monkeypatch, tmp_path):
import utils.models.model_config as mc
repo_dir = tmp_path / "models--org--model"
commit_a = repo_dir / "snapshots" / "commit-a"
commit_b = repo_dir / "snapshots" / "commit-b"
commit_a.mkdir(parents = True)
commit_b.mkdir(parents = True)
(commit_a / "tokenizer_config.json").write_text(
_json.dumps(self._tokenizer_config("<|AUDIO|>", "<|audio_eos|>"))
)
(commit_b / "tokenizer_config.json").write_text(
_json.dumps(self._tokenizer_config("<ordinary-token>"))
)
monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
monkeypatch.setattr(mc, "get_cache_path", lambda *_a, **_k: repo_dir)
monkeypatch.setattr(mc, "_env_offline", lambda: True)
assert _detect_audio_from_tokenizer(
"org/model",
local_files_only = True,
revision = "commit-b",
) == (None, True)
assert _detect_audio_from_tokenizer(
"org/model",
local_files_only = True,
revision = "commit-a",
) == ("csm", True)
def test_remote_tokenizer_read_uses_requested_revision(self, monkeypatch):
import requests
import utils.models.model_config as mc
response = MagicMock(status_code = 200, ok = True)
response.json.return_value = self._tokenizer_config(
"<|AUDIO|>",
"<|audio_eos|>",
)
get = MagicMock(return_value = response)
monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
monkeypatch.setattr(mc, "get_cache_path", lambda *_a, **_k: None)
monkeypatch.setattr(mc, "_env_offline", lambda: False)
monkeypatch.setattr(requests, "get", get)
assert _detect_audio_from_tokenizer("org/model", revision = "refs/pr/7") == ("csm", True)
assert get.call_args.args[0] == (
"https://huggingface.co/org/model/resolve/refs%2Fpr%2F7/tokenizer_config.json"
)
def test_remote_tokenizer_read_keeps_main_without_revision(self, monkeypatch):
import requests
import utils.models.model_config as mc
response = MagicMock(status_code = 200, ok = True)
response.json.return_value = self._tokenizer_config("<|startoftranscript|>")
get = MagicMock(return_value = response)
monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
monkeypatch.setattr(mc, "get_cache_path", lambda *_a, **_k: None)
monkeypatch.setattr(mc, "_env_offline", lambda: False)
monkeypatch.setattr(requests, "get", get)
assert _detect_audio_from_tokenizer("org/model") == ("whisper", True)
assert get.call_args.args[0] == (
"https://huggingface.co/org/model/resolve/main/tokenizer_config.json"
)
class TestEnvOfflineParsing:
"""_env_offline accepts the canonical truthy set (strip+lower, on/true/yes/1); it gates
the requests.get fallback and the cache keys, so 'on' or ' 1 ' must still count as offline."""
def test_truthy_values_recognized(self, monkeypatch):
import utils.models.model_config as mc
for var in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"):
for val in ("1", "true", "TRUE", "yes", "Yes", "on", "ON", " 1 ", " on ", "\ttrue\n"):
monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False)
monkeypatch.setenv(var, val)
assert mc._env_offline() is True, f"{var}={val!r} should be offline"
def test_falsy_values_not_offline(self, monkeypatch):
import utils.models.model_config as mc
monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False)
assert mc._env_offline() is False
for val in ("", "0", "false", "no", "off", "2", "onn"):
monkeypatch.setenv("HF_HUB_OFFLINE", val)
assert mc._env_offline() is False, f"HF_HUB_OFFLINE={val!r} should not be offline"
def _hub_cached_repo(
tmp_path,
repo_id,
files,
sha = "abc123",
):
"""A repo laid out the way the HF hub cache lays one out."""
repo_dir = tmp_path / ("models--" + repo_id.replace("/", "--"))
snapshot = repo_dir / "snapshots" / sha
snapshot.mkdir(parents = True)
for name, text in files.items():
(snapshot / name).write_text(text, encoding = "utf-8")
return repo_dir, snapshot
def _probe_against_cache(
monkeypatch,
repo_dir,
*,
sha = "abc123",
listed = ("config.json",),
remote_config = None,
**kwargs,
):
"""Runs the vision probe on a cached snapshot and logs Hub reads; listed=None means no document."""
import huggingface_hub
import utils.hf_probe as hf_probe
import utils.models.model_config as mc
reads: list = []
monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
monkeypatch.setattr(mc, "get_cache_path", lambda *_a, **_k: repo_dir)
def _info(
model_name,
hf_token = None,
**kw,
):
if listed is None:
raise ConnectionError("no repo document")
return _types.SimpleNamespace(
sha = sha,
siblings = [_types.SimpleNamespace(rfilename = name) for name in listed],
)
monkeypatch.setattr(mc, "_hub_model_info", _info)
def _absent(model_name, filename, **kw):
reads.append(("absent", filename))
return remote_config is None
def _download(**kw):
reads.append(("download", kw.get("filename")))
path = repo_dir / "remote-config.json"
path.write_text(_json.dumps(remote_config), encoding = "utf-8")
return str(path)
monkeypatch.setattr(hf_probe, "hf_file_definitely_absent", _absent)
monkeypatch.setattr(huggingface_hub, "hf_hub_download", _download)
return mc._raw_config_has_vision_config("acme/vlm", **kwargs), reads
@pytest.mark.parametrize(
"cached, expected",
[({"vision_config": {"hidden_size": 8}}, True), ({"model_type": "llama"}, False)],
)
def test_the_current_snapshot_answers_without_fetching_the_file(
tmp_path, monkeypatch, cached, expected
):
"""The cached snapshot answers without fetching config.json; the repo document is still read."""
repo_dir, _ = _hub_cached_repo(tmp_path, "acme/vlm", {"config.json": _json.dumps(cached)})
answer, reads = _probe_against_cache(monkeypatch, repo_dir)
assert answer is expected
assert reads == []
@pytest.mark.parametrize(
"case, files, kwargs",
[
# A repo re-downloaded at a new commit keeps the old snapshot beside the new one.
("stale snapshot", {"config.json": '{"model_type": "llama"}'}, {"sha": "new"}),
("file not downloaded", {"modules.json": "[]"}, {}),
# Offline, or on any failed read, there is nothing to judge the snapshot against.
("no repo document", {"config.json": '{"model_type": "llama"}'}, {"listed": None}),
# The cached snapshot is whichever was downloaded, not the revision asked for.
("pinned revision", {"config.json": '{"model_type": "llama"}'}, {"revision": "commit-a"}),
# Reading the cache authorizes nothing, so a caller who forced anonymity keeps it.
("anonymous", {"config.json": '{"model_type": "llama"}'}, {"hf_token": False}),
],
)
def test_the_hub_still_answers_when_the_snapshot_may_not(
tmp_path, monkeypatch, case, files, kwargs
):
repo_dir, _ = _hub_cached_repo(tmp_path, "acme/vlm", files)
answer, reads = _probe_against_cache(
monkeypatch, repo_dir, remote_config = {"vision_config": {"hidden_size": 8}}, **kwargs
)
assert answer is True, case
assert reads == [("absent", "config.json"), ("download", "config.json")], case
def test_local_files_only_reads_no_repo_document(tmp_path, monkeypatch):
"""The caller asked for no network; resolving the current commit would be one."""
import utils.models.model_config as mc
repo_dir, _ = _hub_cached_repo(
tmp_path, "acme/vlm", {"config.json": _json.dumps({"model_type": "llama"})}
)
reads: list = []
def _info(
model_name,
hf_token = None,
**kw,
):
reads.append(model_name)
raise AssertionError("the repo document must not be read")
monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
monkeypatch.setattr(mc, "get_cache_path", lambda *_a, **_k: repo_dir)
monkeypatch.setattr(mc, "_hub_model_info", _info)
import huggingface_hub
downloads: list = []
def _download(**kw):
downloads.append(kw.get("local_files_only"))
return str(repo_dir / "snapshots" / "abc123" / "config.json")
monkeypatch.setattr(huggingface_hub, "hf_hub_download", _download)
assert mc._raw_config_has_vision_config("acme/vlm", local_files_only = True) is False
assert reads == []
assert downloads == [True]
def test_nothing_cached_costs_no_repo_document(tmp_path, monkeypatch):
"""The document is only worth a round trip when there is a snapshot to judge with it."""
import utils.models.model_config as mc
empty = tmp_path / "models--acme--vlm"
empty.mkdir()
monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
monkeypatch.setattr(mc, "get_cache_path", lambda *_a, **_k: empty)
reads: list = []
def _info(
model_name,
hf_token = None,
**kw,
):
reads.append(model_name)
raise AssertionError("the repo document must not be read")
monkeypatch.setattr(mc, "_hub_model_info", _info)
assert mc._current_cached_snapshot("acme/vlm") is None
assert reads == []
def test_the_current_snapshot_is_the_one_the_repo_document_names(tmp_path, monkeypatch):
"""The helper both probes rest on, exercised rather than stubbed."""
import utils.models.model_config as mc
repo_dir, snapshot = _hub_cached_repo(tmp_path, "acme/vlm", {"config.json": "{}"}, sha = "aaa")
(repo_dir / "snapshots" / "bbb").mkdir()
monkeypatch.setattr(mc, "is_local_path", lambda *_a, **_k: False)
monkeypatch.setattr(mc, "get_cache_path", lambda *_a, **_k: repo_dir)
def _info(
model_name,
hf_token = None,
**kw,
):
return _types.SimpleNamespace(
sha = "aaa", siblings = [_types.SimpleNamespace(rfilename = "config.json")]
)
monkeypatch.setattr(mc, "_hub_model_info", _info)
assert mc._current_cached_snapshot("acme/vlm") == (snapshot, {"config.json"})
assert mc._current_cached_snapshot("acme/vlm", False) is None
def test_a_repo_that_publishes_no_config_is_settled_by_the_document(tmp_path, monkeypatch):
"""A document that omits config.json already settles it; probing the Hub for it wastes a round trip."""
repo_dir, _ = _hub_cached_repo(tmp_path, "acme/vlm", {"model-Q4_K_M.gguf": "x"})
result, reads = _probe_against_cache(monkeypatch, repo_dir, listed = ("model-Q4_K_M.gguf",))
assert result is None
assert reads == []
def test_a_config_the_repo_has_but_the_cache_lacks_still_asks(tmp_path, monkeypatch):
"""Only a document omitting the file settles it; one listing an absent file means fetch it."""
repo_dir, _ = _hub_cached_repo(tmp_path, "acme/vlm", {"model-Q4_K_M.gguf": "x"})
result, reads = _probe_against_cache(
monkeypatch,
repo_dir,
listed = ("model-Q4_K_M.gguf", "config.json"),
remote_config = {"vision_config": {"hidden_size": 8}},
)
assert result is True
assert reads == [("absent", "config.json"), ("download", "config.json")]