Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: read the VLM and audio detection sets from the transformers registry source instead of importing torch (#15688)
Daniel Han94be55a06e28
diff --git a/studio/backend/tests/test_detection_sets_from_source.py b/studio/backend/tests/test_detection_sets_from_source.pynew file mode 100644index 00000000000..0da26dca324--- /dev/null+++ b/studio/backend/tests/test_detection_sets_from_source.py@@ -0,0 +1,244 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Detection sets come from the registry source without importing torch, equal to the import."""++from __future__ import annotations++import importlib.util+import subprocess+import sys+import textwrap+from pathlib import Path++import pytest++import utils.models.model_config as mc++_BACKEND_DIR = Path(__file__).resolve().parent.parent+_HAS_TRANSFORMERS = importlib.util.find_spec("transformers") is not None++[email protected](not _HAS_TRANSFORMERS, reason = "transformers not installed")+def test_building_the_sets_does_not_import_torch_or_the_registry():+ code = textwrap.dedent(+ """+ import sys+ import utils.models.model_config as mc+ sets = mc._detection_sets()+ assert "llava" in sets[0] and "whisper" in sets[2], sets+ heavy = sorted(m for m in ("torch", "transformers.models.auto.modeling_auto") if m in sys.modules)+ print("HEAVY", heavy)+ """+ )+ out = subprocess.run(+ [sys.executable, "-c", code], cwd = _BACKEND_DIR, capture_output = True, text = True, timeout = 600+ )+ assert out.returncode == 0, out.stderr[-2000:]+ assert "HEAVY []" in out.stdout, out.stdout[-500:]+++def _forget_the_imported_registry(monkeypatch):+ # An imported registry makes the build skip the source.+ monkeypatch.delitem(sys.modules, "transformers.models.auto.modeling_auto", raising = False)+++def test_sets_from_source_equal_sets_from_the_import(monkeypatch):+ _forget_the_imported_registry(monkeypatch)+ read = []+ real = mc._mapping_names_from_source+ monkeypatch.setattr(+ mc, "_mapping_names_from_source", lambda path: read.append(real(path)) or read[-1]+ )+ from_source = mc._build_detection_sets()+ if not _HAS_TRANSFORMERS:+ pytest.skip("parity needs transformers: both arms read its registry")+ if mc._transformers_enables_torch():+ assert read and read[0] is not None+ else:+ assert read == []+ _forget_the_imported_registry(monkeypatch)+ monkeypatch.setattr(mc, "_mapping_names_from_source", lambda path: None)+ from_import = mc._build_detection_sets()+ assert from_source == from_import+++def _torch_metadata(+ monkeypatch,+ *,+ found,+ version = "2.5.1",+ floor = "torch>=2.5",+):+ import importlib.metadata++ real_find_spec = importlib.util.find_spec+ real_version = importlib.metadata.version+ real_requires = importlib.metadata.requires+ monkeypatch.delenv("USE_TORCH", raising = False)+ monkeypatch.delenv("USE_TF", raising = False)+ monkeypatch.setattr(+ importlib.util,+ "find_spec",+ lambda name, *a: (object() if found else None)+ if name == "torch"+ else real_find_spec(name, *a),+ )+ monkeypatch.setattr(+ importlib.metadata,+ "version",+ lambda name: version if name == "torch" else real_version(name),+ )+ monkeypatch.setattr(+ importlib.metadata,+ "requires",+ lambda name: [f'{floor}; extra == "torch"', "numpy>=1.17"]+ if name == "transformers"+ else real_requires(name),+ )+++ "condition", ["no torch", "USE_TORCH", "USE_TF", "torch below the floor", "no floor"]+)+def test_the_source_is_not_read_when_transformers_may_refuse_torch(monkeypatch, condition):+ """Without torch the import fails into the curated fallback, which the source would not match."""+ _torch_metadata(+ monkeypatch,+ found = condition != "no torch",+ version = "2.1.0" if condition == "torch below the floor" else "2.5.1",+ floor = "torch" if condition == "no floor" else "torch>=2.5",+ )+ if condition in ("USE_TORCH", "USE_TF"):+ monkeypatch.setenv(condition, "0" if condition == "USE_TORCH" else "1")+ _forget_the_imported_registry(monkeypatch)+ monkeypatch.setattr(+ mc, "_mapping_names_from_source", lambda path: pytest.fail("read the source")+ )+ assert mc._transformers_enables_torch() is False+ mc._build_detection_sets()++[email protected]("version", ["2.5.0", "2.5.1+cu124", "2.14.0.dev20260101"])+def test_the_source_is_read_when_torch_meets_the_declared_floor(monkeypatch, version):+ _torch_metadata(monkeypatch, found = True, version = version)+ assert mc._transformers_enables_torch() is True+++ "env", [{"USE_TF": "0"}, {"USE_TORCH": "1"}, {"USE_TORCH": "auto", "USE_FLAX": "0"}]+)+def test_env_values_that_keep_torch_on_still_read_the_source(monkeypatch, env):+ # `import unsloth` sets USE_TF=0 and USE_FLAX=0 for the processes it starts.+ _torch_metadata(monkeypatch, found = True)+ for key, value in env.items():+ monkeypatch.setenv(key, value)+ assert mc._transformers_enables_torch() is True+++def test_a_declared_torch_ceiling_is_not_a_runtime_gate(monkeypatch):+ # transformers 4.52 declares torch<2.7,>=2.1 for its extras and still loads newer torch.+ _torch_metadata(monkeypatch, found = True, version = "2.14.0", floor = "torch<2.7,>=2.1")+ assert mc._transformers_enables_torch() is True+++def test_an_imported_registry_is_used_instead_of_the_source(monkeypatch):+ import types++ fake = types.ModuleType("transformers.models.auto.modeling_auto")+ fake.MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES = {"fake_vlm": "FakeForImageTextToText"}+ parents = {}+ for name in ("transformers", "transformers.models", "transformers.models.auto"):+ parents[name] = types.ModuleType(name)+ parents[name].__path__ = []+ monkeypatch.setitem(sys.modules, name, parents[name])+ parents["transformers.models.auto"].modeling_auto = fake+ monkeypatch.setitem(sys.modules, "transformers.models.auto.modeling_auto", fake)+ monkeypatch.setattr(+ mc, "_mapping_names_from_source", lambda path: pytest.fail("read the source")+ )+ vlm_types, vlm_classes, _ = mc._build_detection_sets()+ assert "fake_vlm" in vlm_types and "FakeForImageTextToText" in vlm_classes+++_GOOD = """+from collections import OrderedDict+MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES = OrderedDict([("llava", "LlavaForConditionalGeneration")])+MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES = OrderedDict([("blip", ("BlipA", "BlipB"))])+MODEL_FOR_CTC_MAPPING_NAMES = OrderedDict([("wav2vec2", "Wav2Vec2ForCTC")])+MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES = {"whisper": "WhisperForConditionalGeneration"}+MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES = OrderedDict([])+MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES = OrderedDict([("bark", "BarkModel")])+MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES = OrderedDict([("speecht5", "SpeechT5ForTextToSpeech")])+MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES = OrderedDict([("wavlm", "WavLMForXVector")])+MULTIMODAL = OrderedDict([*list(MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES.items()), ("x", "Y")])+LAZY = _LazyAutoMapping(CONFIG_MAPPING_NAMES, MODEL_FOR_CTC_MAPPING_NAMES)+FIRST = MODEL_FOR_CTC_MAPPING_NAMES["wav2vec2"]+"""+++def _read(tmp_path, source: str):+ path = tmp_path / "modeling_auto.py"+ path.write_text(source, encoding = "utf-8")+ return mc._mapping_names_from_source(str(path))+++def test_reader_takes_plain_literals(tmp_path):+ found = _read(tmp_path, _GOOD)+ assert found["MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES"] == {"blip": ("BlipA", "BlipB")}+ assert found["MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES"] == {+ "whisper": "WhisperForConditionalGeneration"+ }+ assert found["MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES"] == {}+++def test_a_name_the_module_never_mentions_reads_as_empty(tmp_path):+ source = "\n".join(l for l in _GOOD.splitlines() if "VISION_2_SEQ" not in l)+ assert _read(tmp_path, source)["MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES"] == {}+++ "extra",+ [+ 'MODEL_FOR_CTC_MAPPING_NAMES.update({"hubert": "HubertForCTC"})',+ 'MODEL_FOR_CTC_MAPPING_NAMES["hubert"] = "HubertForCTC"',+ 'del MODEL_FOR_CTC_MAPPING_NAMES["wav2vec2"]',+ 'MODEL_FOR_CTC_MAPPING_NAMES.pop("wav2vec2")',+ 'MODEL_FOR_CTC_MAPPING_NAMES = OrderedDict([("a", "B")])',+ 'MODEL_FOR_CTC_MAPPING_NAMES |= {"a": "B"}',+ "def f():\n global MODEL_FOR_CTC_MAPPING_NAMES",+ "from .other import MODEL_FOR_CTC_MAPPING_NAMES",+ "for MODEL_FOR_CTC_MAPPING_NAMES in []:\n pass",+ ],+)+def test_reader_declines_when_the_runtime_dict_could_differ(tmp_path, extra):+ assert _read(tmp_path, _GOOD + "\n" + extra + "\n") is None+++ "source",+ [+ _GOOD.replace('OrderedDict([("wav2vec2", "Wav2Vec2ForCTC")])', "_build()"),+ _GOOD.replace('("wav2vec2", "Wav2Vec2ForCTC")', '("wav2vec2", NAME)'),+ _GOOD.replace('("wav2vec2", "Wav2Vec2ForCTC")', '(1, "Wav2Vec2ForCTC")'),+ "\n".join(l for l in _GOOD.splitlines() if "VISION_2_SEQ" not in l)+ + "\ndef __getattr__(name):\n return {}\n",+ "\n".join(l for l in _GOOD.splitlines() if "VISION_2_SEQ" not in l)+ + "\nfrom .other import *\n",+ "\n".join(l for l in _GOOD.splitlines() if l.startswith("MODEL_FOR_VISION_2_SEQ") is False)+ + "\nprint(MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES)\n",+ "def broken(:\n",+ "x = 1\n",+ ],+)+def test_reader_declines_what_it_cannot_prove(tmp_path, source):+ assert _read(tmp_path, source) is None+++def test_a_declined_source_falls_back_to_the_import(monkeypatch):+ _forget_the_imported_registry(monkeypatch)+ calls = []+ monkeypatch.setattr(mc, "_mapping_names_from_source", lambda path: calls.append(path))+ sets = mc._build_detection_sets()+ if _HAS_TRANSFORMERS and mc._transformers_enables_torch():+ assert calls and "llava" in sets[0]diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.pyindex 4058dd99a94..aa303fbe7ef 100644--- a/studio/backend/utils/models/model_config.py+++ b/studio/backend/utils/models/model_config.py@@ -646,14 +646,180 @@ _CURATED_REMOTE_VLM_TYPES = frozenset(_FALLBACK_AUDIO_MODEL_TYPES = frozenset({"csm", "whisper"})+_VLM_MAPPING_ATTRS = (+ "MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES",+ "MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES",+)+_AUDIO_MAPPING_ATTRS = (+ "MODEL_FOR_CTC_MAPPING_NAMES",+ "MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES",+ "MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES",+ "MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES",+ "MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES",+ "MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES",+)+++_READ_ONLY_DICT_METHODS = frozenset({"items", "keys", "values", "get", "copy"})+++def _modeling_auto_source_path() -> Optional[str]:+ """modeling_auto.py of the transformers an import would load now, found without importing it."""+ import importlib.util++ package = sys.modules.get("transformers")+ try:+ if package is not None:+ locations = list(getattr(package, "__path__", None) or [])+ else:+ spec = importlib.util.find_spec("transformers")+ locations = list(spec.submodule_search_locations or []) if spec is not None else []+ except (ImportError, ValueError, AttributeError):+ return None+ if len(locations) != 1:+ return None+ path = os.path.join(locations[0], "models", "auto", "modeling_auto.py")+ return path if os.path.isfile(path) else None+++def _transformers_enables_torch() -> bool:+ """True only when transformers surely enables torch (installed, USE_TORCH / USE_TF, declared+ floor), decided from metadata. False keeps the import path, so no-torch installs still get the+ curated fallback."""+ import importlib.metadata+ import importlib.util++ # transformers 4.x honours these; `import unsloth` sets USE_TF=0, which leaves torch on.+ true_values = {"1", "ON", "YES", "TRUE"}+ if os.environ.get("USE_TORCH", "AUTO").upper() not in true_values | {"AUTO"}:+ return False+ if os.environ.get("USE_TF", "AUTO").upper() in true_values:+ return False+ try:+ from packaging.requirements import Requirement+ from packaging.version import Version++ if importlib.util.find_spec("torch") is None:+ return False+ torch_version = Version(importlib.metadata.version("torch"))+ requirements = [Requirement(r) for r in importlib.metadata.requires("transformers") or []]+ except Exception: # noqa: BLE001+ return False+ # Floors only: ceilings are not runtime gates (4.52 declares torch<2.7, runs on newer torch).+ floors = [+ spec+ for r in requirements+ if r.name == "torch"+ for spec in r.specifier+ if spec.operator in (">=", ">")+ ]+ if not floors:+ return False+ return all(spec.contains(torch_version, prereleases = True) for spec in floors)+++def _mapping_names_from_source(path: str) -> Optional[Dict[str, dict]]:+ """The wanted *_MAPPING_NAMES from modeling_auto.py's source via ast, without importing torch.+ None (caller imports) unless each is bound once at module level to a literal and never+ rebound or mutated."""+ import ast++ wanted = frozenset(_VLM_MAPPING_ATTRS + _AUDIO_MAPPING_ATTRS)+ try:+ with open(path, encoding = "utf-8") as fh:+ tree = ast.parse(fh.read())+ except (OSError, SyntaxError, ValueError, UnicodeDecodeError, MemoryError, RecursionError):+ return None+ found: Dict[str, dict] = {}+ for node in tree.body:+ if not (+ isinstance(node, ast.Assign)+ and len(node.targets) == 1+ and isinstance(node.targets[0], ast.Name)+ and node.targets[0].id in wanted+ ):+ continue+ value = node.value+ if (+ isinstance(value, ast.Call)+ and isinstance(value.func, ast.Name)+ and value.func.id == "OrderedDict"+ and len(value.args) == 1+ and not value.keywords+ ):+ value = value.args[0]+ try:+ literal = ast.literal_eval(value)+ mapping = dict(literal)+ except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError):+ return None+ if node.targets[0].id in found or not all(isinstance(k, str) for k in mapping):+ return None+ found[node.targets[0].id] = mapping+ # Unmentioned names read as {} like getattr(..., None) (VISION_2_SEQ is gone in 5.x), unless+ # __getattr__, a star import or globals() could create them.+ mentioned = {n.id for n in ast.walk(tree) if isinstance(n, ast.Name) and n.id in wanted}+ for node in ast.walk(tree):+ if (+ (isinstance(node, ast.FunctionDef) and node.name == "__getattr__")+ or (isinstance(node, ast.ImportFrom) and any(a.name == "*" for a in node.names))+ or (isinstance(node, ast.Name) and node.id in ("globals", "vars", "setattr", "exec"))+ ):+ if wanted - set(found):+ return None+ if not found:+ return None+ for name in wanted - set(found):+ if name in mentioned:+ return None+ found[name] = {}+ stores: Dict[str, int] = {}+ for node in ast.walk(tree):+ if isinstance(node, ast.Name) and node.id in wanted and not isinstance(node.ctx, ast.Load):+ stores[node.id] = stores.get(node.id, 0) + 1+ elif isinstance(node, (ast.Attribute, ast.Subscript)):+ base = node.value+ if (+ isinstance(base, ast.Name)+ and base.id in wanted+ and not (+ isinstance(node.ctx, ast.Load)+ and (isinstance(node, ast.Subscript) or node.attr in _READ_ONLY_DICT_METHODS)+ )+ ):+ return None+ elif isinstance(node, (ast.Global, ast.Nonlocal)) and wanted.intersection(node.names):+ return None+ elif isinstance(node, (ast.Import, ast.ImportFrom)) and any(+ (alias.asname or alias.name) in wanted for alias in node.names+ ):+ return None+ if any(stores.get(name, 0) != (1 if name in mentioned else 0) for name in wanted):+ return None+ return found++def _build_detection_sets():"""Reads static name dicts from the transformers registry only; no model loads and no code runs."""try:- from transformers.models.auto import modeling_auto as _ma+ source_path = (+ None+ if "transformers.models.auto.modeling_auto" in sys.modules+ or not _transformers_enables_torch()+ else _modeling_auto_source_path()+ )+ from_source = _mapping_names_from_source(source_path) if source_path else None+ if from_source is not None:- def _names(attr):- d = getattr(_ma, attr, None)- return dict(d) if d else {}+ def _names(attr):+ d = from_source[attr]+ return dict(d) if d else {}++ else:+ from transformers.models.auto import modeling_auto as _ma+ def _names(attr):+ d = getattr(_ma, attr, None)+ return dict(d) if d else {}itt = _names("MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES")v2s = _names("MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES")@@ -661,14 +827,7 @@ def _build_detection_sets():vlm_classes = set(itt.values()) | set(v2s.values())audio_types: set = set()- for attr in (- "MODEL_FOR_CTC_MAPPING_NAMES",- "MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES",- "MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES",- "MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES",- "MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES",- "MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES",- ):+ for attr in _AUDIO_MAPPING_ATTRS:audio_types |= set(_names(attr))audio_types |= set(_FALLBACK_AUDIO_MODEL_TYPES)diff --git a/studio/backend/utils/torch_warmup.py b/studio/backend/utils/torch_warmup.pyindex c17930db40e..06c0837d96f 100644--- a/studio/backend/utils/torch_warmup.py+++ b/studio/backend/utils/torch_warmup.py@@ -189,6 +189,11 @@ def _warm_hardware(epoch: Optional[int] = None) -> None:def _warm_transformers() -> None:from utils.models.model_config import _detection_sets_detection_sets()+ # The sets may skip the import now; keep warming it for the first request that loads a model.+ try:+ importlib.import_module("transformers.models.auto.modeling_auto")+ except Exception as exc: # noqa: BLE001+ logger.debug("transformers registry import failed in the warm: %r", exc)def _warm_datasets() -> None: