Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Warn when the installed transformers drops a pre-quantized checkpoint's bnb quant state (#11450)
Daniel Hanb7aca555354c
diff --git a/tests/test_prequantized_vlm_quant_state_guard.py b/tests/test_prequantized_vlm_quant_state_guard.pynew file mode 100644index 00000000000..f46b9edd030--- /dev/null+++ b/tests/test_prequantized_vlm_quant_state_guard.py@@ -0,0 +1,429 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Tests for the pre-quantized quant_state guard (unsloth #9867, #10010, #10017, #10276).++transformers 5.4.0 (PR #44300) merged a submodule's renamings into the parent without+re-scoping them, so a composite checkpoint's bitsandbytes sidecars renamed to keys the+model does not have and were dropped and every quantized Linear loads with+``quant_state = None``. Fixed upstream by PR #45567.++The guard asks the INSTALLED transformers whether it scopes submodule renamings, never a+version: main reported ``5.3.0.dev0`` at the commit that introduced the defect and+``5.6.0.dev0`` at the one that fixed it.++Loaded in isolation, no torch, no GPU, no network.+"""++from __future__ import annotations++import importlib.util+import logging+import sys+import types+from pathlib import Path++import pytest+import tomllib+from packaging.requirements import Requirement++_ROOT = Path(__file__).resolve().parent.parent+_IMPORT_FIXES_PATH = _ROOT / "unsloth" / "import_fixes.py"+_PYPROJECT_PATH = _ROOT / "pyproject.toml"++# Releases that carry the defect, and releases that do not. Used only to document the+# window in the warning text and to keep the pyproject specifier honest -- never as the+# detector's input.+BROKEN_RELEASES = ["5.4.0", "5.5.0", "5.5.1", "5.5.2", "5.5.3", "5.5.4"]+GOOD_RELEASES = ["4.57.6", "5.2.0", "5.3.0", "5.6.0", "5.6.2", "5.14.1", "5.17.0"]+++def _load_import_fixes():+ spec = importlib.util.spec_from_file_location(+ "unsloth_import_fixes_quant_state_under_test", _IMPORT_FIXES_PATH+ )+ assert spec is not None and spec.loader is not None+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module++[email protected](scope = "module")+def import_fixes():+ return _load_import_fixes()+++# Fake transformers builds. Each one is a shape upstream really shipped.+++def _install(+ monkeypatch,+ *,+ conversion_mapping = None,+ core_model_loading = None,+):+ """Put a fake `transformers` in sys.modules for the duration of one test."""+ fake = types.ModuleType("transformers")+ # A real transformers may already be imported, and `from transformers import X` falls+ # back to sys.modules["transformers.X"], so every submodule entry is pinned here --+ # None makes the import raise, which is what "this build does not have it" means.+ for name, submodule in (+ ("conversion_mapping", conversion_mapping),+ ("core_model_loading", core_model_loading),+ ):+ if submodule is not None:+ setattr(fake, name, submodule)+ monkeypatch.setitem(sys.modules, f"transformers.{name}", submodule)+ monkeypatch.setitem(sys.modules, "transformers", fake)+ return fake+++def _module(name, **attrs):+ module = types.ModuleType(name)+ for key, value in attrs.items():+ setattr(module, key, value)+ return module+++def _build_4x(monkeypatch):+ """transformers 4.x: no conversion_mapping module at all, so nothing to correct."""+ _install(monkeypatch)+++def _build_pre_recursion(monkeypatch):+ """5.x before 5.4.0: a conversion_mapping, but no per-submodule extraction."""+ _install(+ monkeypatch,+ conversion_mapping = _module("transformers.conversion_mapping"),+ core_model_loading = _module("transformers.core_model_loading"),+ )+++def _build_broken(monkeypatch):+ """5.4.0 to 5.5.4: recursion, and no way to say where a submodule's mapping belongs."""++ def extract_weight_conversions_for_model(model):+ return []++ class WeightTransform:+ pass++ class PrefixChange:+ pass++ _install(+ monkeypatch,+ conversion_mapping = _module(+ "transformers.conversion_mapping",+ extract_weight_conversions_for_model = extract_weight_conversions_for_model,+ ),+ core_model_loading = _module(+ "transformers.core_model_loading",+ WeightTransform = WeightTransform,+ PrefixChange = PrefixChange,+ ),+ )+++def _build_fixed_model_prefix(monkeypatch):+ """5.6 to 5.9 as PR #45567 first spelled it: a `model_prefix` argument."""++ def extract_weight_conversions_for_model(model, model_prefix = ""):+ return []++ _install(+ monkeypatch,+ conversion_mapping = _module(+ "transformers.conversion_mapping",+ extract_weight_conversions_for_model = extract_weight_conversions_for_model,+ ),+ core_model_loading = _module("transformers.core_model_loading"),+ )+++def _build_fixed_with_submodel_prefix(monkeypatch):+ """The same range, reached through `PrefixChange.with_submodel_prefix`."""++ def extract_weight_conversions_for_model(model):+ return []++ class PrefixChange:+ def with_submodel_prefix(self, prefix):+ return self++ _install(+ monkeypatch,+ conversion_mapping = _module(+ "transformers.conversion_mapping",+ extract_weight_conversions_for_model = extract_weight_conversions_for_model,+ ),+ core_model_loading = _module("transformers.core_model_loading", PrefixChange = PrefixChange),+ )+++def _build_fixed_scope_prefix(monkeypatch):+ """5.10 and later: every transform carries `scope_prefix`."""++ def extract_weight_conversions_for_model(model):+ return []++ class WeightTransform:+ scope_prefix = None++ _install(+ monkeypatch,+ conversion_mapping = _module(+ "transformers.conversion_mapping",+ extract_weight_conversions_for_model = extract_weight_conversions_for_model,+ ),+ core_model_loading = _module(+ "transformers.core_model_loading", WeightTransform = WeightTransform+ ),+ )+++def _build_broken_without_core_model_loading(monkeypatch):+ """The defect with `core_model_loading` unimportable: defensive, not observed.++ Measured on ten real releases both modules need torch and fail together, so the+ signature check is what would have to carry the answer if upstream ever split them.+ """++ def extract_weight_conversions_for_model(model):+ return []++ _install(+ monkeypatch,+ conversion_mapping = _module(+ "transformers.conversion_mapping",+ extract_weight_conversions_for_model = extract_weight_conversions_for_model,+ ),+ core_model_loading = None,+ )+++def _build_fixed_without_core_model_loading(monkeypatch):+ """The same split, fixed: `model_prefix` alone is enough to clear the build."""++ def extract_weight_conversions_for_model(model, model_prefix = ""):+ return []++ _install(+ monkeypatch,+ conversion_mapping = _module(+ "transformers.conversion_mapping",+ extract_weight_conversions_for_model = extract_weight_conversions_for_model,+ ),+ core_model_loading = None,+ )+++def _build_no_transformers(monkeypatch):+ """transformers is not installed."""+ for name in (+ "transformers",+ "transformers.conversion_mapping",+ "transformers.core_model_loading",+ ):+ monkeypatch.setitem(sys.modules, name, None)+++DEFECTIVE_BUILDS = [_build_broken, _build_broken_without_core_model_loading]+HEALTHY_BUILDS = [+ _build_4x,+ _build_pre_recursion,+ _build_fixed_model_prefix,+ _build_fixed_with_submodel_prefix,+ _build_fixed_scope_prefix,+ _build_fixed_without_core_model_loading,+ _build_no_transformers,+]+++# The probe++[email protected]("build", DEFECTIVE_BUILDS, ids = lambda b: b.__name__)+def test_defective_build_is_detected(import_fixes, monkeypatch, build):+ build(monkeypatch)+ assert import_fixes._transformers_rescopes_submodule_prefix_renamings() is False+ assert import_fixes._transformers_drops_prequantized_vlm_quant_state() is True++[email protected]("build", HEALTHY_BUILDS, ids = lambda b: b.__name__)+def test_healthy_build_is_not_flagged(import_fixes, monkeypatch, build):+ build(monkeypatch)+ assert import_fixes._transformers_rescopes_submodule_prefix_renamings() is True+ assert import_fixes._transformers_drops_prequantized_vlm_quant_state() is False+++def test_an_unrecognisable_build_is_left_alone(import_fixes, monkeypatch):+ """Guessing wrong towards "broken" would warn every future build. Answer healthy."""++ class Hostile:+ def __getattr__(self, name):+ raise RuntimeError("no introspection here")++ _install(+ monkeypatch,+ conversion_mapping = _module(+ "transformers.conversion_mapping",+ extract_weight_conversions_for_model = Hostile(),+ ),+ core_model_loading = _module("transformers.core_model_loading"),+ )+ assert import_fixes._transformers_drops_prequantized_vlm_quant_state() is False+++def test_the_detector_never_reads_a_version(import_fixes, monkeypatch):+ """Main was 5.3.0.dev0 when broken and 5.6.0.dev0 when fixed: the answer must not+ move when the reported version does."""+ for reported in ("5.3.0.dev0", "5.6.0.dev0", "5.17.0", "4.57.6", "not-a-version"):+ monkeypatch.setattr(+ import_fixes, "importlib_version", lambda name, reported = reported: reported+ )+ _build_broken(monkeypatch)+ assert import_fixes._transformers_drops_prequantized_vlm_quant_state() is True+ _build_fixed_scope_prefix(monkeypatch)+ assert import_fixes._transformers_drops_prequantized_vlm_quant_state() is False+++def test_a_git_main_build_carrying_the_defect_is_warned(import_fixes, monkeypatch, caplog):+ """5.3.0.dev0 with the defect: the old version window let this through silently."""+ monkeypatch.setattr(import_fixes, "importlib_version", lambda name: "5.3.0.dev0")+ monkeypatch.delenv("UNSLOTH_SKIP_TRANSFORMERS_QUANT_STATE_CHECK", raising = False)+ _build_broken(monkeypatch)+ with caplog.at_level(logging.WARNING):+ import_fixes.check_transformers_prequantized_vlm_quant_state()+ assert "quant_state" in caplog.text+++def test_a_fixed_nightly_is_not_told_it_is_broken(import_fixes, monkeypatch, caplog):+ """5.6.0.dev0 with the fix: the old version window warned this one falsely."""+ monkeypatch.setattr(import_fixes, "importlib_version", lambda name: "5.6.0.dev0")+ monkeypatch.delenv("UNSLOTH_SKIP_TRANSFORMERS_QUANT_STATE_CHECK", raising = False)+ _build_fixed_scope_prefix(monkeypatch)+ with caplog.at_level(logging.WARNING):+ import_fixes.check_transformers_prequantized_vlm_quant_state()+ assert "quant_state" not in caplog.text+++# The warning+++def test_warning_names_the_cause_and_the_remedy(import_fixes, monkeypatch, caplog):+ monkeypatch.setattr(import_fixes, "importlib_version", lambda name: "5.5.4")+ monkeypatch.delenv("UNSLOTH_SKIP_TRANSFORMERS_QUANT_STATE_CHECK", raising = False)+ _build_broken(monkeypatch)+ with caplog.at_level(logging.WARNING):+ import_fixes.check_transformers_prequantized_vlm_quant_state()+ text = caplog.text+ assert "5.5.4" in text+ assert "quant_state" in text+ # It must say the checkpoint is fine: the reported threads told users to regenerate it.+ assert "not be regenerated" in text.lower()+ assert "transformers>=5.6.0" in text+ # And it must not tell the user to do something this repository forbids without+ # saying how: unsloth still caps transformers at 5.5.0.+ assert "--no-deps" in text+++def test_warning_silent_on_a_healthy_build(import_fixes, monkeypatch, caplog):+ monkeypatch.setattr(import_fixes, "importlib_version", lambda name: "5.17.0")+ monkeypatch.delenv("UNSLOTH_SKIP_TRANSFORMERS_QUANT_STATE_CHECK", raising = False)+ _build_fixed_scope_prefix(monkeypatch)+ with caplog.at_level(logging.WARNING):+ import_fixes.check_transformers_prequantized_vlm_quant_state()+ assert "quant_state" not in caplog.text+++def test_env_var_silences_the_warning(import_fixes, monkeypatch, caplog):+ monkeypatch.setattr(import_fixes, "importlib_version", lambda name: "5.5.4")+ monkeypatch.setenv("UNSLOTH_SKIP_TRANSFORMERS_QUANT_STATE_CHECK", "1")+ _build_broken(monkeypatch)+ with caplog.at_level(logging.WARNING):+ import_fixes.check_transformers_prequantized_vlm_quant_state()+ assert "quant_state" not in caplog.text+++def test_missing_transformers_never_raises(import_fixes, monkeypatch, caplog):+ def _boom(name):+ raise ModuleNotFoundError(name)++ monkeypatch.setattr(import_fixes, "importlib_version", _boom)+ monkeypatch.delenv("UNSLOTH_SKIP_TRANSFORMERS_QUANT_STATE_CHECK", raising = False)+ _build_no_transformers(monkeypatch)+ with caplog.at_level(logging.WARNING):+ import_fixes.check_transformers_prequantized_vlm_quant_state()+ assert "quant_state" not in caplog.text+++def test_the_check_is_warn_only(import_fixes, monkeypatch):+ """A text-only or unquantized run must not be broken by this."""+ monkeypatch.delenv("UNSLOTH_SKIP_TRANSFORMERS_QUANT_STATE_CHECK", raising = False)+ for build in DEFECTIVE_BUILDS + HEALTHY_BUILDS:+ build(monkeypatch)+ assert import_fixes.check_transformers_prequantized_vlm_quant_state() is None+++# pyproject+++def _transformers_requirements():+ data = tomllib.loads(_PYPROJECT_PATH.read_text(encoding = "utf-8"))+ found = []+ for group in (data.get("project", {}).get("optional-dependencies", {}) or {}).values():+ for raw in group:+ try:+ req = Requirement(raw)+ except Exception:+ continue+ if req.name == "transformers":+ found.append(req)+ for raw in data.get("project", {}).get("dependencies", []) or []:+ try:+ req = Requirement(raw)+ except Exception:+ continue+ if req.name == "transformers":+ found.append(req)+ return found+++def test_pyproject_still_allows_a_working_version():+ """unsloth_zoo caps transformers at 5.5.0 and `loader.py` needs >= 5.5.0 for Gemma 4,+ so excluding 5.5.0 here resolves DOWN to 5.3.0 and takes Gemma 4 with it."""+ reqs = _transformers_requirements()+ assert reqs, "no transformers requirement found in pyproject.toml"+ for req in reqs:+ assert any(+ req.specifier.contains(version, prereleases = True) for version in GOOD_RELEASES+ ), f"pyproject leaves no working transformers at all: {req}"+ assert req.specifier.contains("5.5.0", prereleases = True), (+ f"pyproject excludes 5.5.0 while unsloth_zoo still caps transformers at "+ f"5.5.0 and Gemma 4 requires >= 5.5.0, so this resolves backwards: {req}"+ )+++# Where the check is called from+++def test_the_check_runs_after_the_torchaudio_guard():+ """The probe imports transformers, and `disable_torchaudio_if_cuda_mismatched` exists+ because reaching torchaudio before it runs takes the whole `import unsloth` down."""+ import ast++ source = (_ROOT / "unsloth" / "_gpu_init.py").read_text(encoding = "utf-8")+ order = []+ for node in ast.parse(source).body:+ if (+ isinstance(node, ast.Expr)+ and isinstance(node.value, ast.Call)+ and isinstance(node.value.func, ast.Name)+ ):+ order.append(node.value.func.id)+ assert "disable_torchaudio_if_cuda_mismatched" in order, order+ assert "check_transformers_prequantized_vlm_quant_state" in order, order+ assert order.index("check_transformers_prequantized_vlm_quant_state") > order.index(+ "disable_torchaudio_if_cuda_mismatched"+ ), "the quant_state check imports transformers and must not precede the torchaudio guard"diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex bc4592c119e..0d18482e1ce 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -24,6 +24,7 @@ from .import_fixes import (fix_message_factory_issue,patch_torch_missing_attribute_error,check_triton_py_ssize_t_clean,+ check_transformers_prequantized_vlm_quant_state,fix_torch_check_is_size,fix_torchao_torch_symbol_skew,propagate_torchao_fix_to_subprocesses,@@ -103,6 +104,10 @@ torchvision_compatibility_check()disable_torchaudio_if_cuda_mismatched()fix_diffusers_warnings()fix_huggingface_hub()+# Below the torchaudio guard, not up with the other version checks: this is the only check+# here that IMPORTS transformers rather than reading its metadata, and that is the ordering+# the guard above exists to prevent.+check_transformers_prequantized_vlm_quant_state()del configure_amdgpu_asic_id_table_pathdel fix_bitsandbytes_rocm_arch_detectiondel disable_broken_causal_conv1d@@ -115,6 +120,7 @@ del propagate_torchao_fix_to_subprocessesdel check_fbgemm_gpu_versiondel check_transformers_dependency_versionsdel check_triton_py_ssize_t_clean+del check_transformers_prequantized_vlm_quant_statedel torchvision_compatibility_checkdel fix_diffusers_warningsdel fix_huggingface_hubdiff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex 79ce03c52f8..10313113169 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -3265,6 +3265,101 @@ def check_triton_py_ssize_t_clean():)+def _transformers_rescopes_submodule_prefix_renamings():+ """True when this build scopes a nested submodule's conversions to where it lives.++ A submodule's mapping is written against ITS OWN key space, so once+ `get_model_conversion_mapping` merges it into the parent (transformers PR #44300) a+ renaming anchored at the start of the key is meaningless; PR #45567 re-scoped them by+ the submodule's dotted path. Three spellings since, any one of which counts: the+ `model_prefix` argument of 5.6 to 5.9, `PrefixChange.with_submodel_prefix` over the+ same range, `scope_prefix` on every transform from 5.10 on.++ Every unknown answers True: a warning is worth nothing if it fires on builds nobody+ can show are broken.+ """+ try:+ from transformers import conversion_mapping+ except Exception:+ return True+ extract = getattr(conversion_mapping, "extract_weight_conversions_for_model", None)+ if extract is None:+ # No per-submodule extraction, so no recursion to correct: every 5.x before 5.4.0.+ return True+ try:+ from transformers import core_model_loading+ except Exception:+ # Separate import, allowed to fail: it needs torch, and the two spellings it carries+ # are extra evidence. The signature check below has to stand on its own.+ core_model_loading = None+ if core_model_loading is not None:+ transform = getattr(core_model_loading, "WeightTransform", None)+ if transform is not None and hasattr(transform, "scope_prefix"):+ return True+ prefix_change = getattr(core_model_loading, "PrefixChange", None)+ if prefix_change is not None and hasattr(prefix_change, "with_submodel_prefix"):+ return True+ try:+ return "model_prefix" in inspect.signature(extract).parameters+ except Exception:+ return True+++def _transformers_drops_prequantized_vlm_quant_state():+ """True when the installed transformers loses bnb-4bit quant_state on composite models.++ The text model's `^model.language_model.` -> `^model.` renaming enters the composite+ model's mapping and runs before the bitsandbytes converter, so `weight` loads as a raw+ packed uint8 while `weight.absmax`, `weight.quant_map`, `weight.nested_absmax`,+ `weight.nested_quant_map` and `weight.quant_state.bitsandbytes__nf4` rename to keys the+ model does not have and are dropped as unexpected. Affected text config types:+ `qwen3_5_text`, `qwen3_5_moe_text`, `gemma3n_text`; flat text-only checkpoints have no+ such prefix, which is why the window went unnoticed.++ Asked of the API and not of a version, because main reported `5.3.0.dev0` at the commit+ that introduced the defect and `5.6.0.dev0` at the one that fixed it, so an interval is+ wrong at both ends on a source install. Among releases it is 5.4.0 and 5.5.0 to 5.5.4.+ """+ return not _transformers_rescopes_submodule_prefix_renamings()+++def check_transformers_prequantized_vlm_quant_state():+ """Warn when transformers will silently drop a pre-quantized VLM's quant_state.++ unsloth #9867, #10010, #10017, #10276 all report `mat1 and mat2 shapes cannot be+ multiplied`, which reads like a corrupt checkpoint and sent reporters off regenerating+ good ones. Warns rather than raises: a run touching only text-only or unquantized+ checkpoints is unaffected and must not break.+ """+ if os.environ.get("UNSLOTH_SKIP_TRANSFORMERS_QUANT_STATE_CHECK", "0").lower() in (+ "1",+ "true",+ ):+ return+ if not _transformers_drops_prequantized_vlm_quant_state():+ return+ try:+ transformers_version = importlib_version("transformers")+ except Exception:+ transformers_version = "unknown"++ logger.warning(+ f"Unsloth: transformers=={transformers_version} drops the bitsandbytes "+ f"quant_state of pre-quantized multimodal checkpoints while loading them, so "+ f"every quantized layer comes back unquantized and the first forward fails "+ f"with\n"+ f" RuntimeError: mat1 and mat2 shapes cannot be multiplied (... and 1x...)\n"+ f"The checkpoint is fine and must not be regenerated. This build scopes no "+ f"submodule conversion mapping, which is the defect transformers PR #44300 "+ f"introduced and PR #45567 fixed; among releases that is 5.4.0 and 5.5.0 to "+ f"5.5.4. Move to a transformers that carries the fix, "+ f'`pip install --no-deps "transformers>=5.6.0"` while unsloth still caps at '+ f"5.5.0, or fall back to 5.3.0 or 4.57.6. Text-only checkpoints are "+ f"unaffected. Set UNSLOTH_SKIP_TRANSFORMERS_QUANT_STATE_CHECK=1 to silence "+ f"this."+ )++# Fix TRL OpenEnv 0.26 NameError: name 'SamplingParams' is not defineddef fix_openenv_no_vllm():spec = importlib.util.find_spec("trl")