Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: read the VLM and audio detection sets from the transformers registry source instead of importing torch (#15688)

Daniel Han94be55a06e28
diff --git a/studio/backend/tests/test_detection_sets_from_source.py b/studio/backend/tests/test_detection_sets_from_source.py
new file mode 100644
index 00000000000..0da26dca324
--- /dev/null
+++ b/studio/backend/tests/test_detection_sets_from_source.py
@@ -0,0 +1,244 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Detection sets come from the registry source without importing torch, equal to the import."""
+
+from __future__ import annotations
+
+import importlib.util
+import subprocess
+import sys
+import textwrap
+from pathlib import Path
+
+import pytest
+
+import utils.models.model_config as mc
+
+_BACKEND_DIR = Path(__file__).resolve().parent.parent
+_HAS_TRANSFORMERS = importlib.util.find_spec("transformers") is not None
+
+
[email protected](not _HAS_TRANSFORMERS, reason = "transformers not installed")
+def test_building_the_sets_does_not_import_torch_or_the_registry():
+ code = textwrap.dedent(
+ """
+ import sys
+ import utils.models.model_config as mc
+ sets = mc._detection_sets()
+ assert "llava" in sets[0] and "whisper" in sets[2], sets
+ heavy = sorted(m for m in ("torch", "transformers.models.auto.modeling_auto") if m in sys.modules)
+ print("HEAVY", heavy)
+ """
+ )
+ out = subprocess.run(
+ [sys.executable, "-c", code], cwd = _BACKEND_DIR, capture_output = True, text = True, timeout = 600
+ )
+ assert out.returncode == 0, out.stderr[-2000:]
+ assert "HEAVY []" in out.stdout, out.stdout[-500:]
+
+
+def _forget_the_imported_registry(monkeypatch):
+ # An imported registry makes the build skip the source.
+ monkeypatch.delitem(sys.modules, "transformers.models.auto.modeling_auto", raising = False)
+
+
+def test_sets_from_source_equal_sets_from_the_import(monkeypatch):
+ _forget_the_imported_registry(monkeypatch)
+ read = []
+ real = mc._mapping_names_from_source
+ monkeypatch.setattr(
+ mc, "_mapping_names_from_source", lambda path: read.append(real(path)) or read[-1]
+ )
+ from_source = mc._build_detection_sets()
+ if not _HAS_TRANSFORMERS:
+ pytest.skip("parity needs transformers: both arms read its registry")
+ if mc._transformers_enables_torch():
+ assert read and read[0] is not None
+ else:
+ assert read == []
+ _forget_the_imported_registry(monkeypatch)
+ monkeypatch.setattr(mc, "_mapping_names_from_source", lambda path: None)
+ from_import = mc._build_detection_sets()
+ assert from_source == from_import
+
+
+def _torch_metadata(
+ monkeypatch,
+ *,
+ found,
+ version = "2.5.1",
+ floor = "torch>=2.5",
+):
+ import importlib.metadata
+
+ real_find_spec = importlib.util.find_spec
+ real_version = importlib.metadata.version
+ real_requires = importlib.metadata.requires
+ monkeypatch.delenv("USE_TORCH", raising = False)
+ monkeypatch.delenv("USE_TF", raising = False)
+ monkeypatch.setattr(
+ importlib.util,
+ "find_spec",
+ lambda name, *a: (object() if found else None)
+ if name == "torch"
+ else real_find_spec(name, *a),
+ )
+ monkeypatch.setattr(
+ importlib.metadata,
+ "version",
+ lambda name: version if name == "torch" else real_version(name),
+ )
+ monkeypatch.setattr(
+ importlib.metadata,
+ "requires",
+ lambda name: [f'{floor}; extra == "torch"', "numpy>=1.17"]
+ if name == "transformers"
+ else real_requires(name),
+ )
+
+
+ "condition", ["no torch", "USE_TORCH", "USE_TF", "torch below the floor", "no floor"]
+)
+def test_the_source_is_not_read_when_transformers_may_refuse_torch(monkeypatch, condition):
+ """Without torch the import fails into the curated fallback, which the source would not match."""
+ _torch_metadata(
+ monkeypatch,
+ found = condition != "no torch",
+ version = "2.1.0" if condition == "torch below the floor" else "2.5.1",
+ floor = "torch" if condition == "no floor" else "torch>=2.5",
+ )
+ if condition in ("USE_TORCH", "USE_TF"):
+ monkeypatch.setenv(condition, "0" if condition == "USE_TORCH" else "1")
+ _forget_the_imported_registry(monkeypatch)
+ monkeypatch.setattr(
+ mc, "_mapping_names_from_source", lambda path: pytest.fail("read the source")
+ )
+ assert mc._transformers_enables_torch() is False
+ mc._build_detection_sets()
+
+
[email protected]("version", ["2.5.0", "2.5.1+cu124", "2.14.0.dev20260101"])
+def test_the_source_is_read_when_torch_meets_the_declared_floor(monkeypatch, version):
+ _torch_metadata(monkeypatch, found = True, version = version)
+ assert mc._transformers_enables_torch() is True
+
+
+ "env", [{"USE_TF": "0"}, {"USE_TORCH": "1"}, {"USE_TORCH": "auto", "USE_FLAX": "0"}]
+)
+def test_env_values_that_keep_torch_on_still_read_the_source(monkeypatch, env):
+ # `import unsloth` sets USE_TF=0 and USE_FLAX=0 for the processes it starts.
+ _torch_metadata(monkeypatch, found = True)
+ for key, value in env.items():
+ monkeypatch.setenv(key, value)
+ assert mc._transformers_enables_torch() is True
+
+
+def test_a_declared_torch_ceiling_is_not_a_runtime_gate(monkeypatch):
+ # transformers 4.52 declares torch<2.7,>=2.1 for its extras and still loads newer torch.
+ _torch_metadata(monkeypatch, found = True, version = "2.14.0", floor = "torch<2.7,>=2.1")
+ assert mc._transformers_enables_torch() is True
+
+
+def test_an_imported_registry_is_used_instead_of_the_source(monkeypatch):
+ import types
+
+ fake = types.ModuleType("transformers.models.auto.modeling_auto")
+ fake.MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES = {"fake_vlm": "FakeForImageTextToText"}
+ parents = {}
+ for name in ("transformers", "transformers.models", "transformers.models.auto"):
+ parents[name] = types.ModuleType(name)
+ parents[name].__path__ = []
+ monkeypatch.setitem(sys.modules, name, parents[name])
+ parents["transformers.models.auto"].modeling_auto = fake
+ monkeypatch.setitem(sys.modules, "transformers.models.auto.modeling_auto", fake)
+ monkeypatch.setattr(
+ mc, "_mapping_names_from_source", lambda path: pytest.fail("read the source")
+ )
+ vlm_types, vlm_classes, _ = mc._build_detection_sets()
+ assert "fake_vlm" in vlm_types and "FakeForImageTextToText" in vlm_classes
+
+
+_GOOD = """
+from collections import OrderedDict
+MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES = OrderedDict([("llava", "LlavaForConditionalGeneration")])
+MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES = OrderedDict([("blip", ("BlipA", "BlipB"))])
+MODEL_FOR_CTC_MAPPING_NAMES = OrderedDict([("wav2vec2", "Wav2Vec2ForCTC")])
+MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES = {"whisper": "WhisperForConditionalGeneration"}
+MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES = OrderedDict([])
+MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES = OrderedDict([("bark", "BarkModel")])
+MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES = OrderedDict([("speecht5", "SpeechT5ForTextToSpeech")])
+MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES = OrderedDict([("wavlm", "WavLMForXVector")])
+MULTIMODAL = OrderedDict([*list(MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES.items()), ("x", "Y")])
+LAZY = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_CTC_MAPPING_NAMES)
+FIRST = MODEL_FOR_CTC_MAPPING_NAMES["wav2vec2"]
+"""
+
+
+def _read(tmp_path, source: str):
+ path = tmp_path / "modeling_auto.py"
+ path.write_text(source, encoding = "utf-8")
+ return mc._mapping_names_from_source(str(path))
+
+
+def test_reader_takes_plain_literals(tmp_path):
+ found = _read(tmp_path, _GOOD)
+ assert found["MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES"] == {"blip": ("BlipA", "BlipB")}
+ assert found["MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES"] == {
+ "whisper": "WhisperForConditionalGeneration"
+ }
+ assert found["MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES"] == {}
+
+
+def test_a_name_the_module_never_mentions_reads_as_empty(tmp_path):
+ source = "\n".join(l for l in _GOOD.splitlines() if "VISION_2_SEQ" not in l)
+ assert _read(tmp_path, source)["MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES"] == {}
+
+
+ "extra",
+ [
+ 'MODEL_FOR_CTC_MAPPING_NAMES.update({"hubert": "HubertForCTC"})',
+ 'MODEL_FOR_CTC_MAPPING_NAMES["hubert"] = "HubertForCTC"',
+ 'del MODEL_FOR_CTC_MAPPING_NAMES["wav2vec2"]',
+ 'MODEL_FOR_CTC_MAPPING_NAMES.pop("wav2vec2")',
+ 'MODEL_FOR_CTC_MAPPING_NAMES = OrderedDict([("a", "B")])',
+ 'MODEL_FOR_CTC_MAPPING_NAMES |= {"a": "B"}',
+ "def f():\n global MODEL_FOR_CTC_MAPPING_NAMES",
+ "from .other import MODEL_FOR_CTC_MAPPING_NAMES",
+ "for MODEL_FOR_CTC_MAPPING_NAMES in []:\n pass",
+ ],
+)
+def test_reader_declines_when_the_runtime_dict_could_differ(tmp_path, extra):
+ assert _read(tmp_path, _GOOD + "\n" + extra + "\n") is None
+
+
+ "source",
+ [
+ _GOOD.replace('OrderedDict([("wav2vec2", "Wav2Vec2ForCTC")])', "_build()"),
+ _GOOD.replace('("wav2vec2", "Wav2Vec2ForCTC")', '("wav2vec2", NAME)'),
+ _GOOD.replace('("wav2vec2", "Wav2Vec2ForCTC")', '(1, "Wav2Vec2ForCTC")'),
+ "\n".join(l for l in _GOOD.splitlines() if "VISION_2_SEQ" not in l)
+ + "\ndef __getattr__(name):\n return {}\n",
+ "\n".join(l for l in _GOOD.splitlines() if "VISION_2_SEQ" not in l)
+ + "\nfrom .other import *\n",
+ "\n".join(l for l in _GOOD.splitlines() if l.startswith("MODEL_FOR_VISION_2_SEQ") is False)
+ + "\nprint(MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES)\n",
+ "def broken(:\n",
+ "x = 1\n",
+ ],
+)
+def test_reader_declines_what_it_cannot_prove(tmp_path, source):
+ assert _read(tmp_path, source) is None
+
+
+def test_a_declined_source_falls_back_to_the_import(monkeypatch):
+ _forget_the_imported_registry(monkeypatch)
+ calls = []
+ monkeypatch.setattr(mc, "_mapping_names_from_source", lambda path: calls.append(path))
+ sets = mc._build_detection_sets()
+ if _HAS_TRANSFORMERS and mc._transformers_enables_torch():
+ assert calls and "llava" in sets[0]
diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py
index 4058dd99a94..aa303fbe7ef 100644
--- a/studio/backend/utils/models/model_config.py
+++ b/studio/backend/utils/models/model_config.py
@@ -646,14 +646,180 @@ _CURATED_REMOTE_VLM_TYPES = frozenset(
_FALLBACK_AUDIO_MODEL_TYPES = frozenset({"csm", "whisper"})
+_VLM_MAPPING_ATTRS = (
+ "MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES",
+ "MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES",
+)
+_AUDIO_MAPPING_ATTRS = (
+ "MODEL_FOR_CTC_MAPPING_NAMES",
+ "MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES",
+ "MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES",
+ "MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES",
+ "MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES",
+ "MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES",
+)
+
+
+_READ_ONLY_DICT_METHODS = frozenset({"items", "keys", "values", "get", "copy"})
+
+
+def _modeling_auto_source_path() -> Optional[str]:
+ """modeling_auto.py of the transformers an import would load now, found without importing it."""
+ import importlib.util
+
+ package = sys.modules.get("transformers")
+ try:
+ if package is not None:
+ locations = list(getattr(package, "__path__", None) or [])
+ else:
+ spec = importlib.util.find_spec("transformers")
+ locations = list(spec.submodule_search_locations or []) if spec is not None else []
+ except (ImportError, ValueError, AttributeError):
+ return None
+ if len(locations) != 1:
+ return None
+ path = os.path.join(locations[0], "models", "auto", "modeling_auto.py")
+ return path if os.path.isfile(path) else None
+
+
+def _transformers_enables_torch() -> bool:
+ """True only when transformers surely enables torch (installed, USE_TORCH / USE_TF, declared
+ floor), decided from metadata. False keeps the import path, so no-torch installs still get the
+ curated fallback."""
+ import importlib.metadata
+ import importlib.util
+
+ # transformers 4.x honours these; `import unsloth` sets USE_TF=0, which leaves torch on.
+ true_values = {"1", "ON", "YES", "TRUE"}
+ if os.environ.get("USE_TORCH", "AUTO").upper() not in true_values | {"AUTO"}:
+ return False
+ if os.environ.get("USE_TF", "AUTO").upper() in true_values:
+ return False
+ try:
+ from packaging.requirements import Requirement
+ from packaging.version import Version
+
+ if importlib.util.find_spec("torch") is None:
+ return False
+ torch_version = Version(importlib.metadata.version("torch"))
+ requirements = [Requirement(r) for r in importlib.metadata.requires("transformers") or []]
+ except Exception: # noqa: BLE001
+ return False
+ # Floors only: ceilings are not runtime gates (4.52 declares torch<2.7, runs on newer torch).
+ floors = [
+ spec
+ for r in requirements
+ if r.name == "torch"
+ for spec in r.specifier
+ if spec.operator in (">=", ">")
+ ]
+ if not floors:
+ return False
+ return all(spec.contains(torch_version, prereleases = True) for spec in floors)
+
+
+def _mapping_names_from_source(path: str) -> Optional[Dict[str, dict]]:
+ """The wanted *_MAPPING_NAMES from modeling_auto.py's source via ast, without importing torch.
+ None (caller imports) unless each is bound once at module level to a literal and never
+ rebound or mutated."""
+ import ast
+
+ wanted = frozenset(_VLM_MAPPING_ATTRS + _AUDIO_MAPPING_ATTRS)
+ try:
+ with open(path, encoding = "utf-8") as fh:
+ tree = ast.parse(fh.read())
+ except (OSError, SyntaxError, ValueError, UnicodeDecodeError, MemoryError, RecursionError):
+ return None
+ found: Dict[str, dict] = {}
+ for node in tree.body:
+ if not (
+ isinstance(node, ast.Assign)
+ and len(node.targets) == 1
+ and isinstance(node.targets[0], ast.Name)
+ and node.targets[0].id in wanted
+ ):
+ continue
+ value = node.value
+ if (
+ isinstance(value, ast.Call)
+ and isinstance(value.func, ast.Name)
+ and value.func.id == "OrderedDict"
+ and len(value.args) == 1
+ and not value.keywords
+ ):
+ value = value.args[0]
+ try:
+ literal = ast.literal_eval(value)
+ mapping = dict(literal)
+ except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError):
+ return None
+ if node.targets[0].id in found or not all(isinstance(k, str) for k in mapping):
+ return None
+ found[node.targets[0].id] = mapping
+ # Unmentioned names read as {} like getattr(..., None) (VISION_2_SEQ is gone in 5.x), unless
+ # __getattr__, a star import or globals() could create them.
+ mentioned = {n.id for n in ast.walk(tree) if isinstance(n, ast.Name) and n.id in wanted}
+ for node in ast.walk(tree):
+ if (
+ (isinstance(node, ast.FunctionDef) and node.name == "__getattr__")
+ or (isinstance(node, ast.ImportFrom) and any(a.name == "*" for a in node.names))
+ or (isinstance(node, ast.Name) and node.id in ("globals", "vars", "setattr", "exec"))
+ ):
+ if wanted - set(found):
+ return None
+ if not found:
+ return None
+ for name in wanted - set(found):
+ if name in mentioned:
+ return None
+ found[name] = {}
+ stores: Dict[str, int] = {}
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Name) and node.id in wanted and not isinstance(node.ctx, ast.Load):
+ stores[node.id] = stores.get(node.id, 0) + 1
+ elif isinstance(node, (ast.Attribute, ast.Subscript)):
+ base = node.value
+ if (
+ isinstance(base, ast.Name)
+ and base.id in wanted
+ and not (
+ isinstance(node.ctx, ast.Load)
+ and (isinstance(node, ast.Subscript) or node.attr in _READ_ONLY_DICT_METHODS)
+ )
+ ):
+ return None
+ elif isinstance(node, (ast.Global, ast.Nonlocal)) and wanted.intersection(node.names):
+ return None
+ elif isinstance(node, (ast.Import, ast.ImportFrom)) and any(
+ (alias.asname or alias.name) in wanted for alias in node.names
+ ):
+ return None
+ if any(stores.get(name, 0) != (1 if name in mentioned else 0) for name in wanted):
+ return None
+ return found
+
+
def _build_detection_sets():
"""Reads static name dicts from the transformers registry only; no model loads and no code runs."""
try:
- from transformers.models.auto import modeling_auto as _ma
+ source_path = (
+ None
+ if "transformers.models.auto.modeling_auto" in sys.modules
+ or not _transformers_enables_torch()
+ else _modeling_auto_source_path()
+ )
+ from_source = _mapping_names_from_source(source_path) if source_path else None
+ if from_source is not None:
- def _names(attr):
- d = getattr(_ma, attr, None)
- return dict(d) if d else {}
+ def _names(attr):
+ d = from_source[attr]
+ return dict(d) if d else {}
+
+ else:
+ from transformers.models.auto import modeling_auto as _ma
+ def _names(attr):
+ d = getattr(_ma, attr, None)
+ return dict(d) if d else {}
itt = _names("MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES")
v2s = _names("MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES")
@@ -661,14 +827,7 @@ def _build_detection_sets():
vlm_classes = set(itt.values()) | set(v2s.values())
audio_types: set = set()
- for attr in (
- "MODEL_FOR_CTC_MAPPING_NAMES",
- "MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES",
- "MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES",
- "MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES",
- "MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES",
- "MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES",
- ):
+ for attr in _AUDIO_MAPPING_ATTRS:
audio_types |= set(_names(attr))
audio_types |= set(_FALLBACK_AUDIO_MODEL_TYPES)
diff --git a/studio/backend/utils/torch_warmup.py b/studio/backend/utils/torch_warmup.py
index c17930db40e..06c0837d96f 100644
--- a/studio/backend/utils/torch_warmup.py
+++ b/studio/backend/utils/torch_warmup.py
@@ -189,6 +189,11 @@ def _warm_hardware(epoch: Optional[int] = None) -> None:
def _warm_transformers() -> None:
from utils.models.model_config import _detection_sets
_detection_sets()
+ # The sets may skip the import now; keep warming it for the first request that loads a model.
+ try:
+ importlib.import_module("transformers.models.auto.modeling_auto")
+ except Exception as exc: # noqa: BLE001
+ logger.debug("transformers registry import failed in the warm: %r", exc)
def _warm_datasets() -> None: