Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Warn when the installed transformers drops a pre-quantized checkpoint's bnb quant state (#11450)

Daniel Hanb7aca555354c
diff --git a/tests/test_prequantized_vlm_quant_state_guard.py b/tests/test_prequantized_vlm_quant_state_guard.py
new file mode 100644
index 00000000000..f46b9edd030
--- /dev/null
+++ b/tests/test_prequantized_vlm_quant_state_guard.py
@@ -0,0 +1,429 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Tests for the pre-quantized quant_state guard (unsloth #9867, #10010, #10017, #10276).
+
+transformers 5.4.0 (PR #44300) merged a submodule's renamings into the parent without
+re-scoping them, so a composite checkpoint's bitsandbytes sidecars renamed to keys the
+model does not have and were dropped and every quantized Linear loads with
+``quant_state = None``. Fixed upstream by PR #45567.
+
+The guard asks the INSTALLED transformers whether it scopes submodule renamings, never a
+version: main reported ``5.3.0.dev0`` at the commit that introduced the defect and
+``5.6.0.dev0`` at the one that fixed it.
+
+Loaded in isolation, no torch, no GPU, no network.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import logging
+import sys
+import types
+from pathlib import Path
+
+import pytest
+import tomllib
+from packaging.requirements import Requirement
+
+_ROOT = Path(__file__).resolve().parent.parent
+_IMPORT_FIXES_PATH = _ROOT / "unsloth" / "import_fixes.py"
+_PYPROJECT_PATH = _ROOT / "pyproject.toml"
+
+# Releases that carry the defect, and releases that do not. Used only to document the
+# window in the warning text and to keep the pyproject specifier honest -- never as the
+# detector's input.
+BROKEN_RELEASES = ["5.4.0", "5.5.0", "5.5.1", "5.5.2", "5.5.3", "5.5.4"]
+GOOD_RELEASES = ["4.57.6", "5.2.0", "5.3.0", "5.6.0", "5.6.2", "5.14.1", "5.17.0"]
+
+
+def _load_import_fixes():
+ spec = importlib.util.spec_from_file_location(
+ "unsloth_import_fixes_quant_state_under_test", _IMPORT_FIXES_PATH
+ )
+ assert spec is not None and spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
[email protected](scope = "module")
+def import_fixes():
+ return _load_import_fixes()
+
+
+# Fake transformers builds. Each one is a shape upstream really shipped.
+
+
+def _install(
+ monkeypatch,
+ *,
+ conversion_mapping = None,
+ core_model_loading = None,
+):
+ """Put a fake `transformers` in sys.modules for the duration of one test."""
+ fake = types.ModuleType("transformers")
+ # A real transformers may already be imported, and `from transformers import X` falls
+ # back to sys.modules["transformers.X"], so every submodule entry is pinned here --
+ # None makes the import raise, which is what "this build does not have it" means.
+ for name, submodule in (
+ ("conversion_mapping", conversion_mapping),
+ ("core_model_loading", core_model_loading),
+ ):
+ if submodule is not None:
+ setattr(fake, name, submodule)
+ monkeypatch.setitem(sys.modules, f"transformers.{name}", submodule)
+ monkeypatch.setitem(sys.modules, "transformers", fake)
+ return fake
+
+
+def _module(name, **attrs):
+ module = types.ModuleType(name)
+ for key, value in attrs.items():
+ setattr(module, key, value)
+ return module
+
+
+def _build_4x(monkeypatch):
+ """transformers 4.x: no conversion_mapping module at all, so nothing to correct."""
+ _install(monkeypatch)
+
+
+def _build_pre_recursion(monkeypatch):
+ """5.x before 5.4.0: a conversion_mapping, but no per-submodule extraction."""
+ _install(
+ monkeypatch,
+ conversion_mapping = _module("transformers.conversion_mapping"),
+ core_model_loading = _module("transformers.core_model_loading"),
+ )
+
+
+def _build_broken(monkeypatch):
+ """5.4.0 to 5.5.4: recursion, and no way to say where a submodule's mapping belongs."""
+
+ def extract_weight_conversions_for_model(model):
+ return []
+
+ class WeightTransform:
+ pass
+
+ class PrefixChange:
+ pass
+
+ _install(
+ monkeypatch,
+ conversion_mapping = _module(
+ "transformers.conversion_mapping",
+ extract_weight_conversions_for_model = extract_weight_conversions_for_model,
+ ),
+ core_model_loading = _module(
+ "transformers.core_model_loading",
+ WeightTransform = WeightTransform,
+ PrefixChange = PrefixChange,
+ ),
+ )
+
+
+def _build_fixed_model_prefix(monkeypatch):
+ """5.6 to 5.9 as PR #45567 first spelled it: a `model_prefix` argument."""
+
+ def extract_weight_conversions_for_model(model, model_prefix = ""):
+ return []
+
+ _install(
+ monkeypatch,
+ conversion_mapping = _module(
+ "transformers.conversion_mapping",
+ extract_weight_conversions_for_model = extract_weight_conversions_for_model,
+ ),
+ core_model_loading = _module("transformers.core_model_loading"),
+ )
+
+
+def _build_fixed_with_submodel_prefix(monkeypatch):
+ """The same range, reached through `PrefixChange.with_submodel_prefix`."""
+
+ def extract_weight_conversions_for_model(model):
+ return []
+
+ class PrefixChange:
+ def with_submodel_prefix(self, prefix):
+ return self
+
+ _install(
+ monkeypatch,
+ conversion_mapping = _module(
+ "transformers.conversion_mapping",
+ extract_weight_conversions_for_model = extract_weight_conversions_for_model,
+ ),
+ core_model_loading = _module("transformers.core_model_loading", PrefixChange = PrefixChange),
+ )
+
+
+def _build_fixed_scope_prefix(monkeypatch):
+ """5.10 and later: every transform carries `scope_prefix`."""
+
+ def extract_weight_conversions_for_model(model):
+ return []
+
+ class WeightTransform:
+ scope_prefix = None
+
+ _install(
+ monkeypatch,
+ conversion_mapping = _module(
+ "transformers.conversion_mapping",
+ extract_weight_conversions_for_model = extract_weight_conversions_for_model,
+ ),
+ core_model_loading = _module(
+ "transformers.core_model_loading", WeightTransform = WeightTransform
+ ),
+ )
+
+
+def _build_broken_without_core_model_loading(monkeypatch):
+ """The defect with `core_model_loading` unimportable: defensive, not observed.
+
+ Measured on ten real releases both modules need torch and fail together, so the
+ signature check is what would have to carry the answer if upstream ever split them.
+ """
+
+ def extract_weight_conversions_for_model(model):
+ return []
+
+ _install(
+ monkeypatch,
+ conversion_mapping = _module(
+ "transformers.conversion_mapping",
+ extract_weight_conversions_for_model = extract_weight_conversions_for_model,
+ ),
+ core_model_loading = None,
+ )
+
+
+def _build_fixed_without_core_model_loading(monkeypatch):
+ """The same split, fixed: `model_prefix` alone is enough to clear the build."""
+
+ def extract_weight_conversions_for_model(model, model_prefix = ""):
+ return []
+
+ _install(
+ monkeypatch,
+ conversion_mapping = _module(
+ "transformers.conversion_mapping",
+ extract_weight_conversions_for_model = extract_weight_conversions_for_model,
+ ),
+ core_model_loading = None,
+ )
+
+
+def _build_no_transformers(monkeypatch):
+ """transformers is not installed."""
+ for name in (
+ "transformers",
+ "transformers.conversion_mapping",
+ "transformers.core_model_loading",
+ ):
+ monkeypatch.setitem(sys.modules, name, None)
+
+
+DEFECTIVE_BUILDS = [_build_broken, _build_broken_without_core_model_loading]
+HEALTHY_BUILDS = [
+ _build_4x,
+ _build_pre_recursion,
+ _build_fixed_model_prefix,
+ _build_fixed_with_submodel_prefix,
+ _build_fixed_scope_prefix,
+ _build_fixed_without_core_model_loading,
+ _build_no_transformers,
+]
+
+
+# The probe
+
+
[email protected]("build", DEFECTIVE_BUILDS, ids = lambda b: b.__name__)
+def test_defective_build_is_detected(import_fixes, monkeypatch, build):
+ build(monkeypatch)
+ assert import_fixes._transformers_rescopes_submodule_prefix_renamings() is False
+ assert import_fixes._transformers_drops_prequantized_vlm_quant_state() is True
+
+
[email protected]("build", HEALTHY_BUILDS, ids = lambda b: b.__name__)
+def test_healthy_build_is_not_flagged(import_fixes, monkeypatch, build):
+ build(monkeypatch)
+ assert import_fixes._transformers_rescopes_submodule_prefix_renamings() is True
+ assert import_fixes._transformers_drops_prequantized_vlm_quant_state() is False
+
+
+def test_an_unrecognisable_build_is_left_alone(import_fixes, monkeypatch):
+ """Guessing wrong towards "broken" would warn every future build. Answer healthy."""
+
+ class Hostile:
+ def __getattr__(self, name):
+ raise RuntimeError("no introspection here")
+
+ _install(
+ monkeypatch,
+ conversion_mapping = _module(
+ "transformers.conversion_mapping",
+ extract_weight_conversions_for_model = Hostile(),
+ ),
+ core_model_loading = _module("transformers.core_model_loading"),
+ )
+ assert import_fixes._transformers_drops_prequantized_vlm_quant_state() is False
+
+
+def test_the_detector_never_reads_a_version(import_fixes, monkeypatch):
+ """Main was 5.3.0.dev0 when broken and 5.6.0.dev0 when fixed: the answer must not
+ move when the reported version does."""
+ for reported in ("5.3.0.dev0", "5.6.0.dev0", "5.17.0", "4.57.6", "not-a-version"):
+ monkeypatch.setattr(
+ import_fixes, "importlib_version", lambda name, reported = reported: reported
+ )
+ _build_broken(monkeypatch)
+ assert import_fixes._transformers_drops_prequantized_vlm_quant_state() is True
+ _build_fixed_scope_prefix(monkeypatch)
+ assert import_fixes._transformers_drops_prequantized_vlm_quant_state() is False
+
+
+def test_a_git_main_build_carrying_the_defect_is_warned(import_fixes, monkeypatch, caplog):
+ """5.3.0.dev0 with the defect: the old version window let this through silently."""
+ monkeypatch.setattr(import_fixes, "importlib_version", lambda name: "5.3.0.dev0")
+ monkeypatch.delenv("UNSLOTH_SKIP_TRANSFORMERS_QUANT_STATE_CHECK", raising = False)
+ _build_broken(monkeypatch)
+ with caplog.at_level(logging.WARNING):
+ import_fixes.check_transformers_prequantized_vlm_quant_state()
+ assert "quant_state" in caplog.text
+
+
+def test_a_fixed_nightly_is_not_told_it_is_broken(import_fixes, monkeypatch, caplog):
+ """5.6.0.dev0 with the fix: the old version window warned this one falsely."""
+ monkeypatch.setattr(import_fixes, "importlib_version", lambda name: "5.6.0.dev0")
+ monkeypatch.delenv("UNSLOTH_SKIP_TRANSFORMERS_QUANT_STATE_CHECK", raising = False)
+ _build_fixed_scope_prefix(monkeypatch)
+ with caplog.at_level(logging.WARNING):
+ import_fixes.check_transformers_prequantized_vlm_quant_state()
+ assert "quant_state" not in caplog.text
+
+
+# The warning
+
+
+def test_warning_names_the_cause_and_the_remedy(import_fixes, monkeypatch, caplog):
+ monkeypatch.setattr(import_fixes, "importlib_version", lambda name: "5.5.4")
+ monkeypatch.delenv("UNSLOTH_SKIP_TRANSFORMERS_QUANT_STATE_CHECK", raising = False)
+ _build_broken(monkeypatch)
+ with caplog.at_level(logging.WARNING):
+ import_fixes.check_transformers_prequantized_vlm_quant_state()
+ text = caplog.text
+ assert "5.5.4" in text
+ assert "quant_state" in text
+ # It must say the checkpoint is fine: the reported threads told users to regenerate it.
+ assert "not be regenerated" in text.lower()
+ assert "transformers>=5.6.0" in text
+ # And it must not tell the user to do something this repository forbids without
+ # saying how: unsloth still caps transformers at 5.5.0.
+ assert "--no-deps" in text
+
+
+def test_warning_silent_on_a_healthy_build(import_fixes, monkeypatch, caplog):
+ monkeypatch.setattr(import_fixes, "importlib_version", lambda name: "5.17.0")
+ monkeypatch.delenv("UNSLOTH_SKIP_TRANSFORMERS_QUANT_STATE_CHECK", raising = False)
+ _build_fixed_scope_prefix(monkeypatch)
+ with caplog.at_level(logging.WARNING):
+ import_fixes.check_transformers_prequantized_vlm_quant_state()
+ assert "quant_state" not in caplog.text
+
+
+def test_env_var_silences_the_warning(import_fixes, monkeypatch, caplog):
+ monkeypatch.setattr(import_fixes, "importlib_version", lambda name: "5.5.4")
+ monkeypatch.setenv("UNSLOTH_SKIP_TRANSFORMERS_QUANT_STATE_CHECK", "1")
+ _build_broken(monkeypatch)
+ with caplog.at_level(logging.WARNING):
+ import_fixes.check_transformers_prequantized_vlm_quant_state()
+ assert "quant_state" not in caplog.text
+
+
+def test_missing_transformers_never_raises(import_fixes, monkeypatch, caplog):
+ def _boom(name):
+ raise ModuleNotFoundError(name)
+
+ monkeypatch.setattr(import_fixes, "importlib_version", _boom)
+ monkeypatch.delenv("UNSLOTH_SKIP_TRANSFORMERS_QUANT_STATE_CHECK", raising = False)
+ _build_no_transformers(monkeypatch)
+ with caplog.at_level(logging.WARNING):
+ import_fixes.check_transformers_prequantized_vlm_quant_state()
+ assert "quant_state" not in caplog.text
+
+
+def test_the_check_is_warn_only(import_fixes, monkeypatch):
+ """A text-only or unquantized run must not be broken by this."""
+ monkeypatch.delenv("UNSLOTH_SKIP_TRANSFORMERS_QUANT_STATE_CHECK", raising = False)
+ for build in DEFECTIVE_BUILDS + HEALTHY_BUILDS:
+ build(monkeypatch)
+ assert import_fixes.check_transformers_prequantized_vlm_quant_state() is None
+
+
+# pyproject
+
+
+def _transformers_requirements():
+ data = tomllib.loads(_PYPROJECT_PATH.read_text(encoding = "utf-8"))
+ found = []
+ for group in (data.get("project", {}).get("optional-dependencies", {}) or {}).values():
+ for raw in group:
+ try:
+ req = Requirement(raw)
+ except Exception:
+ continue
+ if req.name == "transformers":
+ found.append(req)
+ for raw in data.get("project", {}).get("dependencies", []) or []:
+ try:
+ req = Requirement(raw)
+ except Exception:
+ continue
+ if req.name == "transformers":
+ found.append(req)
+ return found
+
+
+def test_pyproject_still_allows_a_working_version():
+ """unsloth_zoo caps transformers at 5.5.0 and `loader.py` needs >= 5.5.0 for Gemma 4,
+ so excluding 5.5.0 here resolves DOWN to 5.3.0 and takes Gemma 4 with it."""
+ reqs = _transformers_requirements()
+ assert reqs, "no transformers requirement found in pyproject.toml"
+ for req in reqs:
+ assert any(
+ req.specifier.contains(version, prereleases = True) for version in GOOD_RELEASES
+ ), f"pyproject leaves no working transformers at all: {req}"
+ assert req.specifier.contains("5.5.0", prereleases = True), (
+ f"pyproject excludes 5.5.0 while unsloth_zoo still caps transformers at "
+ f"5.5.0 and Gemma 4 requires >= 5.5.0, so this resolves backwards: {req}"
+ )
+
+
+# Where the check is called from
+
+
+def test_the_check_runs_after_the_torchaudio_guard():
+ """The probe imports transformers, and `disable_torchaudio_if_cuda_mismatched` exists
+ because reaching torchaudio before it runs takes the whole `import unsloth` down."""
+ import ast
+
+ source = (_ROOT / "unsloth" / "_gpu_init.py").read_text(encoding = "utf-8")
+ order = []
+ for node in ast.parse(source).body:
+ if (
+ isinstance(node, ast.Expr)
+ and isinstance(node.value, ast.Call)
+ and isinstance(node.value.func, ast.Name)
+ ):
+ order.append(node.value.func.id)
+ assert "disable_torchaudio_if_cuda_mismatched" in order, order
+ assert "check_transformers_prequantized_vlm_quant_state" in order, order
+ assert order.index("check_transformers_prequantized_vlm_quant_state") > order.index(
+ "disable_torchaudio_if_cuda_mismatched"
+ ), "the quant_state check imports transformers and must not precede the torchaudio guard"
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index bc4592c119e..0d18482e1ce 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -24,6 +24,7 @@ from .import_fixes import (
fix_message_factory_issue,
patch_torch_missing_attribute_error,
check_triton_py_ssize_t_clean,
+ check_transformers_prequantized_vlm_quant_state,
fix_torch_check_is_size,
fix_torchao_torch_symbol_skew,
propagate_torchao_fix_to_subprocesses,
@@ -103,6 +104,10 @@ torchvision_compatibility_check()
disable_torchaudio_if_cuda_mismatched()
fix_diffusers_warnings()
fix_huggingface_hub()
+# Below the torchaudio guard, not up with the other version checks: this is the only check
+# here that IMPORTS transformers rather than reading its metadata, and that is the ordering
+# the guard above exists to prevent.
+check_transformers_prequantized_vlm_quant_state()
del configure_amdgpu_asic_id_table_path
del fix_bitsandbytes_rocm_arch_detection
del disable_broken_causal_conv1d
@@ -115,6 +120,7 @@ del propagate_torchao_fix_to_subprocesses
del check_fbgemm_gpu_version
del check_transformers_dependency_versions
del check_triton_py_ssize_t_clean
+del check_transformers_prequantized_vlm_quant_state
del torchvision_compatibility_check
del fix_diffusers_warnings
del fix_huggingface_hub
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 79ce03c52f8..10313113169 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -3265,6 +3265,101 @@ def check_triton_py_ssize_t_clean():
)
+def _transformers_rescopes_submodule_prefix_renamings():
+ """True when this build scopes a nested submodule's conversions to where it lives.
+
+ A submodule's mapping is written against ITS OWN key space, so once
+ `get_model_conversion_mapping` merges it into the parent (transformers PR #44300) a
+ renaming anchored at the start of the key is meaningless; PR #45567 re-scoped them by
+ the submodule's dotted path. Three spellings since, any one of which counts: the
+ `model_prefix` argument of 5.6 to 5.9, `PrefixChange.with_submodel_prefix` over the
+ same range, `scope_prefix` on every transform from 5.10 on.
+
+ Every unknown answers True: a warning is worth nothing if it fires on builds nobody
+ can show are broken.
+ """
+ try:
+ from transformers import conversion_mapping
+ except Exception:
+ return True
+ extract = getattr(conversion_mapping, "extract_weight_conversions_for_model", None)
+ if extract is None:
+ # No per-submodule extraction, so no recursion to correct: every 5.x before 5.4.0.
+ return True
+ try:
+ from transformers import core_model_loading
+ except Exception:
+ # Separate import, allowed to fail: it needs torch, and the two spellings it carries
+ # are extra evidence. The signature check below has to stand on its own.
+ core_model_loading = None
+ if core_model_loading is not None:
+ transform = getattr(core_model_loading, "WeightTransform", None)
+ if transform is not None and hasattr(transform, "scope_prefix"):
+ return True
+ prefix_change = getattr(core_model_loading, "PrefixChange", None)
+ if prefix_change is not None and hasattr(prefix_change, "with_submodel_prefix"):
+ return True
+ try:
+ return "model_prefix" in inspect.signature(extract).parameters
+ except Exception:
+ return True
+
+
+def _transformers_drops_prequantized_vlm_quant_state():
+ """True when the installed transformers loses bnb-4bit quant_state on composite models.
+
+ The text model's `^model.language_model.` -> `^model.` renaming enters the composite
+ model's mapping and runs before the bitsandbytes converter, so `weight` loads as a raw
+ packed uint8 while `weight.absmax`, `weight.quant_map`, `weight.nested_absmax`,
+ `weight.nested_quant_map` and `weight.quant_state.bitsandbytes__nf4` rename to keys the
+ model does not have and are dropped as unexpected. Affected text config types:
+ `qwen3_5_text`, `qwen3_5_moe_text`, `gemma3n_text`; flat text-only checkpoints have no
+ such prefix, which is why the window went unnoticed.
+
+ Asked of the API and not of a version, because main reported `5.3.0.dev0` at the commit
+ that introduced the defect and `5.6.0.dev0` at the one that fixed it, so an interval is
+ wrong at both ends on a source install. Among releases it is 5.4.0 and 5.5.0 to 5.5.4.
+ """
+ return not _transformers_rescopes_submodule_prefix_renamings()
+
+
+def check_transformers_prequantized_vlm_quant_state():
+ """Warn when transformers will silently drop a pre-quantized VLM's quant_state.
+
+ unsloth #9867, #10010, #10017, #10276 all report `mat1 and mat2 shapes cannot be
+ multiplied`, which reads like a corrupt checkpoint and sent reporters off regenerating
+ good ones. Warns rather than raises: a run touching only text-only or unquantized
+ checkpoints is unaffected and must not break.
+ """
+ if os.environ.get("UNSLOTH_SKIP_TRANSFORMERS_QUANT_STATE_CHECK", "0").lower() in (
+ "1",
+ "true",
+ ):
+ return
+ if not _transformers_drops_prequantized_vlm_quant_state():
+ return
+ try:
+ transformers_version = importlib_version("transformers")
+ except Exception:
+ transformers_version = "unknown"
+
+ logger.warning(
+ f"Unsloth: transformers=={transformers_version} drops the bitsandbytes "
+ f"quant_state of pre-quantized multimodal checkpoints while loading them, so "
+ f"every quantized layer comes back unquantized and the first forward fails "
+ f"with\n"
+ f" RuntimeError: mat1 and mat2 shapes cannot be multiplied (... and 1x...)\n"
+ f"The checkpoint is fine and must not be regenerated. This build scopes no "
+ f"submodule conversion mapping, which is the defect transformers PR #44300 "
+ f"introduced and PR #45567 fixed; among releases that is 5.4.0 and 5.5.0 to "
+ f"5.5.4. Move to a transformers that carries the fix, "
+ f'`pip install --no-deps "transformers>=5.6.0"` while unsloth still caps at '
+ f"5.5.0, or fall back to 5.3.0 or 4.57.6. Text-only checkpoints are "
+ f"unaffected. Set UNSLOTH_SKIP_TRANSFORMERS_QUANT_STATE_CHECK=1 to silence "
+ f"this."
+ )
+
+
# Fix TRL OpenEnv 0.26 NameError: name 'SamplingParams' is not defined
def fix_openenv_no_vllm():
spec = importlib.util.find_spec("trl")